<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html lang="en">
<head>
<title>Source code</title>
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">
</head>
<body>
<div class="sourceContainer">
<pre><span class="sourceLineNo">001</span><a name="line.1"></a>
<span class="sourceLineNo">002</span>/*<a name="line.2"></a>
<span class="sourceLineNo">003</span> * Copyright (C) 2014 Archie L. Cobbs. All rights reserved.<a name="line.3"></a>
<span class="sourceLineNo">004</span> *<a name="line.4"></a>
<span class="sourceLineNo">005</span> * $Id: RaftKVDatabase.html 939 2015-05-29 01:16:17Z archie.cobbs $<a name="line.5"></a>
<span class="sourceLineNo">006</span> */<a name="line.6"></a>
<span class="sourceLineNo">007</span><a name="line.7"></a>
<span class="sourceLineNo">008</span>package org.jsimpledb.kv.raft;<a name="line.8"></a>
<span class="sourceLineNo">009</span><a name="line.9"></a>
<span class="sourceLineNo">010</span>import com.google.common.base.Function;<a name="line.10"></a>
<span class="sourceLineNo">011</span>import com.google.common.base.Preconditions;<a name="line.11"></a>
<span class="sourceLineNo">012</span>import com.google.common.base.Predicate;<a name="line.12"></a>
<span class="sourceLineNo">013</span>import com.google.common.base.Predicates;<a name="line.13"></a>
<span class="sourceLineNo">014</span>import com.google.common.collect.Iterables;<a name="line.14"></a>
<span class="sourceLineNo">015</span>import com.google.common.primitives.Bytes;<a name="line.15"></a>
<span class="sourceLineNo">016</span><a name="line.16"></a>
<span class="sourceLineNo">017</span>import java.io.BufferedInputStream;<a name="line.17"></a>
<span class="sourceLineNo">018</span>import java.io.ByteArrayInputStream;<a name="line.18"></a>
<span class="sourceLineNo">019</span>import java.io.ByteArrayOutputStream;<a name="line.19"></a>
<span class="sourceLineNo">020</span>import java.io.DataInputStream;<a name="line.20"></a>
<span class="sourceLineNo">021</span>import java.io.DataOutputStream;<a name="line.21"></a>
<span class="sourceLineNo">022</span>import java.io.File;<a name="line.22"></a>
<span class="sourceLineNo">023</span>import java.io.FileInputStream;<a name="line.23"></a>
<span class="sourceLineNo">024</span>import java.io.IOException;<a name="line.24"></a>
<span class="sourceLineNo">025</span>import java.nio.ByteBuffer;<a name="line.25"></a>
<span class="sourceLineNo">026</span>import java.nio.channels.FileChannel;<a name="line.26"></a>
<span class="sourceLineNo">027</span>import java.nio.file.Files;<a name="line.27"></a>
<span class="sourceLineNo">028</span>import java.nio.file.StandardCopyOption;<a name="line.28"></a>
<span class="sourceLineNo">029</span>import java.nio.file.StandardOpenOption;<a name="line.29"></a>
<span class="sourceLineNo">030</span>import java.security.SecureRandom;<a name="line.30"></a>
<span class="sourceLineNo">031</span>import java.util.AbstractMap;<a name="line.31"></a>
<span class="sourceLineNo">032</span>import java.util.ArrayList;<a name="line.32"></a>
<span class="sourceLineNo">033</span>import java.util.Arrays;<a name="line.33"></a>
<span class="sourceLineNo">034</span>import java.util.Collections;<a name="line.34"></a>
<span class="sourceLineNo">035</span>import java.util.HashMap;<a name="line.35"></a>
<span class="sourceLineNo">036</span>import java.util.HashSet;<a name="line.36"></a>
<span class="sourceLineNo">037</span>import java.util.Iterator;<a name="line.37"></a>
<span class="sourceLineNo">038</span>import java.util.LinkedHashSet;<a name="line.38"></a>
<span class="sourceLineNo">039</span>import java.util.List;<a name="line.39"></a>
<span class="sourceLineNo">040</span>import java.util.Map;<a name="line.40"></a>
<span class="sourceLineNo">041</span>import java.util.NavigableSet;<a name="line.41"></a>
<span class="sourceLineNo">042</span>import java.util.NoSuchElementException;<a name="line.42"></a>
<span class="sourceLineNo">043</span>import java.util.concurrent.ExecutionException;<a name="line.43"></a>
<span class="sourceLineNo">044</span>import java.util.concurrent.Executors;<a name="line.44"></a>
<span class="sourceLineNo">045</span>import java.util.concurrent.RejectedExecutionException;<a name="line.45"></a>
<span class="sourceLineNo">046</span>import java.util.concurrent.ScheduledExecutorService;<a name="line.46"></a>
<span class="sourceLineNo">047</span>import java.util.concurrent.ScheduledFuture;<a name="line.47"></a>
<span class="sourceLineNo">048</span>import java.util.concurrent.ThreadFactory;<a name="line.48"></a>
<span class="sourceLineNo">049</span>import java.util.concurrent.TimeUnit;<a name="line.49"></a>
<span class="sourceLineNo">050</span>import java.util.regex.Pattern;<a name="line.50"></a>
<span class="sourceLineNo">051</span><a name="line.51"></a>
<span class="sourceLineNo">052</span>import javax.annotation.PostConstruct;<a name="line.52"></a>
<span class="sourceLineNo">053</span>import javax.annotation.PreDestroy;<a name="line.53"></a>
<span class="sourceLineNo">054</span><a name="line.54"></a>
<span class="sourceLineNo">055</span>import org.dellroad.stuff.io.ByteBufferInputStream;<a name="line.55"></a>
<span class="sourceLineNo">056</span>import org.dellroad.stuff.io.ByteBufferOutputStream;<a name="line.56"></a>
<span class="sourceLineNo">057</span>import org.dellroad.stuff.java.TimedWait;<a name="line.57"></a>
<span class="sourceLineNo">058</span>import org.iq80.leveldb.Options;<a name="line.58"></a>
<span class="sourceLineNo">059</span>import org.iq80.leveldb.impl.Iq80DBFactory;<a name="line.59"></a>
<span class="sourceLineNo">060</span>import org.jsimpledb.kv.CloseableKVStore;<a name="line.60"></a>
<span class="sourceLineNo">061</span>import org.jsimpledb.kv.KVDatabase;<a name="line.61"></a>
<span class="sourceLineNo">062</span>import org.jsimpledb.kv.KVStore;<a name="line.62"></a>
<span class="sourceLineNo">063</span>import org.jsimpledb.kv.KVTransactionException;<a name="line.63"></a>
<span class="sourceLineNo">064</span>import org.jsimpledb.kv.KeyRange;<a name="line.64"></a>
<span class="sourceLineNo">065</span>import org.jsimpledb.kv.KeyRanges;<a name="line.65"></a>
<span class="sourceLineNo">066</span>import org.jsimpledb.kv.RetryTransactionException;<a name="line.66"></a>
<span class="sourceLineNo">067</span>import org.jsimpledb.kv.StaleTransactionException;<a name="line.67"></a>
<span class="sourceLineNo">068</span>import org.jsimpledb.kv.leveldb.LevelDBKVStore;<a name="line.68"></a>
<span class="sourceLineNo">069</span>import org.jsimpledb.kv.mvcc.AtomicKVStore;<a name="line.69"></a>
<span class="sourceLineNo">070</span>import org.jsimpledb.kv.mvcc.MutableView;<a name="line.70"></a>
<span class="sourceLineNo">071</span>import org.jsimpledb.kv.mvcc.Mutations;<a name="line.71"></a>
<span class="sourceLineNo">072</span>import org.jsimpledb.kv.mvcc.Reads;<a name="line.72"></a>
<span class="sourceLineNo">073</span>import org.jsimpledb.kv.mvcc.Writes;<a name="line.73"></a>
<span class="sourceLineNo">074</span>import org.jsimpledb.kv.raft.msg.AppendRequest;<a name="line.74"></a>
<span class="sourceLineNo">075</span>import org.jsimpledb.kv.raft.msg.AppendResponse;<a name="line.75"></a>
<span class="sourceLineNo">076</span>import org.jsimpledb.kv.raft.msg.CommitRequest;<a name="line.76"></a>
<span class="sourceLineNo">077</span>import org.jsimpledb.kv.raft.msg.CommitResponse;<a name="line.77"></a>
<span class="sourceLineNo">078</span>import org.jsimpledb.kv.raft.msg.GrantVote;<a name="line.78"></a>
<span class="sourceLineNo">079</span>import org.jsimpledb.kv.raft.msg.InstallSnapshot;<a name="line.79"></a>
<span class="sourceLineNo">080</span>import org.jsimpledb.kv.raft.msg.Message;<a name="line.80"></a>
<span class="sourceLineNo">081</span>import org.jsimpledb.kv.raft.msg.MessageSwitch;<a name="line.81"></a>
<span class="sourceLineNo">082</span>import org.jsimpledb.kv.raft.msg.RequestVote;<a name="line.82"></a>
<span class="sourceLineNo">083</span>import org.jsimpledb.kv.raft.net.Network;<a name="line.83"></a>
<span class="sourceLineNo">084</span>import org.jsimpledb.kv.raft.net.TCPNetwork;<a name="line.84"></a>
<span class="sourceLineNo">085</span>import org.jsimpledb.kv.util.PrefixKVStore;<a name="line.85"></a>
<span class="sourceLineNo">086</span>import org.jsimpledb.util.ByteUtil;<a name="line.86"></a>
<span class="sourceLineNo">087</span>import org.jsimpledb.util.LongEncoder;<a name="line.87"></a>
<span class="sourceLineNo">088</span>import org.slf4j.Logger;<a name="line.88"></a>
<span class="sourceLineNo">089</span>import org.slf4j.LoggerFactory;<a name="line.89"></a>
<span class="sourceLineNo">090</span><a name="line.90"></a>
<span class="sourceLineNo">091</span>/**<a name="line.91"></a>
<span class="sourceLineNo">092</span> * A distributed {@link org.jsimpledb.kv.KVDatabase} based on the Raft consensus algorithm.<a name="line.92"></a>
<span class="sourceLineNo">093</span> *<a name="line.93"></a>
<span class="sourceLineNo">094</span> * &lt;p&gt;<a name="line.94"></a>
<span class="sourceLineNo">095</span> * Raft defines a distributed consensus algorithm for maintaining a shared state machine.<a name="line.95"></a>
<span class="sourceLineNo">096</span> * Each Raft node maintains a complete copy of the state machine. Cluster nodes elect a<a name="line.96"></a>
<span class="sourceLineNo">097</span> * leader who manages updates to the state machine and provides for consistent readds.<a name="line.97"></a>
<span class="sourceLineNo">098</span> * As long as as a node is part of a majority, the state machine is fully operational.<a name="line.98"></a>
<span class="sourceLineNo">099</span> *<a name="line.99"></a>
<span class="sourceLineNo">100</span> * &lt;p&gt;<a name="line.100"></a>
<span class="sourceLineNo">101</span> * {@link RaftKVDatabase} turns<a name="line.101"></a>
<span class="sourceLineNo">102</span> * this into a transactional key/value database with linearizable ACID semantics:<a name="line.102"></a>
<span class="sourceLineNo">103</span> *  &lt;ul&gt;<a name="line.103"></a>
<span class="sourceLineNo">104</span> *  &lt;li&gt;The Raft state machine is the key/value store data.&lt;/li&gt;<a name="line.104"></a>
<span class="sourceLineNo">105</span> *  &lt;li&gt;Unapplied log entries are stored on disk as serialized {@link Writes}, and also cached in memory.&lt;/li&gt;<a name="line.105"></a>
<span class="sourceLineNo">106</span> *  &lt;li&gt;On leaders only, the {@link Writes} associated with some number of the most recently applied log entries<a name="line.106"></a>
<span class="sourceLineNo">107</span> *      are cached in memory for time &lt;i&gt;T&lt;sub&gt;max&lt;/sub&gt;&lt;/i&gt; since creation, where &lt;i&gt;T&lt;sub&gt;max&lt;/sub&gt;&lt;/i&gt; is the<a name="line.107"></a>
<span class="sourceLineNo">108</span> *      maximum supported transaction duration (see below for how these cached {@link Writes} are used).<a name="line.108"></a>
<span class="sourceLineNo">109</span> *      This caching is subject to &lt;i&gt;M&lt;sub&gt;max&lt;/sub&gt;&lt;/i&gt;, which limits the total memory used by memory-cached<a name="line.109"></a>
<span class="sourceLineNo">110</span> *      {@link Writes}; if reached, these cached {@link Writes} are discarded early.&lt;/li&gt;<a name="line.110"></a>
<span class="sourceLineNo">111</span> *  &lt;li&gt;Concurrent transactions are supported through a simple optimistic locking MVCC scheme (same as used by<a name="line.111"></a>
<span class="sourceLineNo">112</span> *      {@link org.jsimpledb.kv.mvcc.SnapshotKVDatabase}):<a name="line.112"></a>
<span class="sourceLineNo">113</span> *      &lt;ul&gt;<a name="line.113"></a>
<span class="sourceLineNo">114</span> *      &lt;li&gt;Transactions execute locally until commit time, using a {@link MutableView} to collect mutations.<a name="line.114"></a>
<span class="sourceLineNo">115</span> *          The {@link MutableView} is based on the local node's most recent log entry (whether committed or not);<a name="line.115"></a>
<span class="sourceLineNo">116</span> *          this is called the &lt;i&gt;base term and index&lt;/i&gt; for the transaction.&lt;/li&gt;<a name="line.116"></a>
<span class="sourceLineNo">117</span> *      &lt;li&gt;On commit, the transaction's {@link Reads}, {@link Writes}, and base index and term are<a name="line.117"></a>
<span class="sourceLineNo">118</span> *          {@linkplain CommitRequest sent} to the leader.&lt;/li&gt;<a name="line.118"></a>
<span class="sourceLineNo">119</span> *      &lt;li&gt;The leader confirms that the log entry corresponding to the transaction's base index and term either:<a name="line.119"></a>
<span class="sourceLineNo">120</span> *          &lt;ul&gt;<a name="line.120"></a>
<span class="sourceLineNo">121</span> *          &lt;li&gt;Is still present in its own log and not yet applied to the state machine (in which case the<a name="line.121"></a>
<span class="sourceLineNo">122</span> *              {@link Writes} are available with the log entry); or&lt;/li&gt;<a name="line.122"></a>
<span class="sourceLineNo">123</span> *          &lt;li&gt;Corresponds to an already applied log entry whose {@link Writes} are still cached in memory.&lt;/li&gt;<a name="line.123"></a>
<span class="sourceLineNo">124</span> *          &lt;/ul&gt;<a name="line.124"></a>
<span class="sourceLineNo">125</span> *          If this is not the case, then the transaction's base log entry is too old (older than &lt;i&gt;T&lt;sub&gt;max&lt;/sub&gt;&lt;/i&gt;,<a name="line.125"></a>
<span class="sourceLineNo">126</span> *          or was committed early due to memory pressure), and so the transaction is rejected with a<a name="line.126"></a>
<span class="sourceLineNo">127</span> *          {@link RetryTransactionException}. Otherwise, the {@link Writes} associated with all log entries<a name="line.127"></a>
<span class="sourceLineNo">128</span> *          after the transaction's base log entry are available.<a name="line.128"></a>
<span class="sourceLineNo">129</span> *      &lt;li&gt;The leader confirms that the {@link Writes} associated with log entries created after the transaction's<a name="line.129"></a>
<span class="sourceLineNo">130</span> *          base log entry do not create linearizability violations when {@linkplain Reads#isConflict compared against}<a name="line.130"></a>
<span class="sourceLineNo">131</span> *          the transaction's {@link Reads}. If so, the transaction is rejected with a retry.&lt;/li&gt;<a name="line.131"></a>
<span class="sourceLineNo">132</span> *      &lt;li&gt;The leader adds a new log entry consisting of the transaction's {@link Writes} to its log.<a name="line.132"></a>
<span class="sourceLineNo">133</span> *          The associated term and index become the transaction's &lt;i&gt;commit term and index&lt;/i&gt;; the leader<a name="line.133"></a>
<span class="sourceLineNo">134</span> *          then {@linkplain CommitResponse replies} to the follower with this information.&lt;/li&gt;<a name="line.134"></a>
<span class="sourceLineNo">135</span> *      &lt;li&gt;When the follower sees a log entry appear in its log matching the transaction's commit term and index, and<a name="line.135"></a>
<span class="sourceLineNo">136</span> *          that log entry has been committed (in the Raft sense), the transaction is also committed.&lt;/li&gt;<a name="line.136"></a>
<span class="sourceLineNo">137</span> *      &lt;li&gt;As an optimization, when the leader sends a log entry to the same follower who committed the corresponding<a name="line.137"></a>
<span class="sourceLineNo">138</span> *          transaction in the first place, only the transaction ID is sent, because the follower already has the data.&lt;/li&gt;<a name="line.138"></a>
<span class="sourceLineNo">139</span> *      &lt;/ul&gt;<a name="line.139"></a>
<span class="sourceLineNo">140</span> *  &lt;/li&gt;<a name="line.140"></a>
<span class="sourceLineNo">141</span> *  &lt;li&gt;For transactions occurring on a leader, the logic is the same except of course no network communication occurs.&lt;/li&gt;<a name="line.141"></a>
<span class="sourceLineNo">142</span> *  &lt;li&gt;For read-only transactions, the leader does not create a new log entry; instead, the transaction's commit<a name="line.142"></a>
<span class="sourceLineNo">143</span> *      term and index are set to the term and index of the last entry in the leader's log. The leader also calculates its<a name="line.143"></a>
<span class="sourceLineNo">144</span> *      current "leader lease timeout", which is the earliest time at which it is possible for another leader to be elected.<a name="line.144"></a>
<span class="sourceLineNo">145</span> *      This is calculated as the time in the past at which the leader sent {@link AppendRequest}'s to a majority of followers<a name="line.145"></a>
<span class="sourceLineNo">146</span> *      who have since responded, plus the {@linkplain #setMinElectionTimeout minimum election timeout}, minus a small adjustment<a name="line.146"></a>
<span class="sourceLineNo">147</span> *      for possible clock drift (this assumes all nodes have the same minimum election timeout configured). If the current<a name="line.147"></a>
<span class="sourceLineNo">148</span> *      time is prior to the leader lease timeout, the transaction may be committed immediately; otherwise, the current time<a name="line.148"></a>
<span class="sourceLineNo">149</span> *      is returned to the follower as minimum required leader lease timeout before the transaction may be committed.&lt;/li&gt;<a name="line.149"></a>
<span class="sourceLineNo">150</span> *  &lt;li&gt;Included with each {@link AppendRequest} is the leader's current timestamp and lease timeout, so followers can commit<a name="line.150"></a>
<span class="sourceLineNo">151</span> *      any waiting read-only transactions. Leaders keep track of which followers are waiting on which leader lease<a name="line.151"></a>
<span class="sourceLineNo">152</span> *      timeout values, and when the leader lease timeout advances to allow a follower to commit a transaction, the follower<a name="line.152"></a>
<span class="sourceLineNo">153</span> *      is notified with an immediate {@link AppendRequest} update.&lt;/li&gt;<a name="line.153"></a>
<span class="sourceLineNo">154</span> *  &lt;li&gt;A weaker consistency guarantee for read-only transactions (serializable but not linearizable) is also possible; see<a name="line.154"></a>
<span class="sourceLineNo">155</span> *      {@link RaftKVTransaction#setReadOnlySnapshot RaftKVTransaction.setReadOnlySnapshot()}.&lt;/li&gt;<a name="line.155"></a>
<span class="sourceLineNo">156</span> *  &lt;/ul&gt;<a name="line.156"></a>
<span class="sourceLineNo">157</span> *<a name="line.157"></a>
<span class="sourceLineNo">158</span> * &lt;p&gt;&lt;b&gt;Limitations&lt;/b&gt;&lt;/p&gt;<a name="line.158"></a>
<span class="sourceLineNo">159</span> *<a name="line.159"></a>
<span class="sourceLineNo">160</span> * &lt;ul&gt;<a name="line.160"></a>
<span class="sourceLineNo">161</span> *  &lt;li&gt;A transaction's mutations must fit in memory.&lt;/li&gt;<a name="line.161"></a>
<span class="sourceLineNo">162</span> *  &lt;li&gt;An {@link AtomicKVStore} is required to store local persistent state;<a name="line.162"></a>
<span class="sourceLineNo">163</span> *      if none is configured, a {@link LevelDBKVStore} is used.&lt;/li&gt;<a name="line.163"></a>
<span class="sourceLineNo">164</span> *  &lt;li&gt;All nodes must be configured with the same {@linkplain #setMinElectionTimeout minimum election timeout}.&lt;/li&gt;<a name="line.164"></a>
<span class="sourceLineNo">165</span> *  &lt;li&gt;Due to the optimistic locking approach used, this implementation will perform poorly when there is a high<a name="line.165"></a>
<span class="sourceLineNo">166</span> *      rate of conflicting transactions; the result will be many transaction retries.&lt;/li&gt;<a name="line.166"></a>
<span class="sourceLineNo">167</span> *  &lt;li&gt;Performance will suffer when the amount of data associated with a typical transaction cannot be delivered<a name="line.167"></a>
<span class="sourceLineNo">168</span> *      quickly and reliably over the network.&lt;/li&gt;<a name="line.168"></a>
<span class="sourceLineNo">169</span> * &lt;/ul&gt;<a name="line.169"></a>
<span class="sourceLineNo">170</span> *<a name="line.170"></a>
<span class="sourceLineNo">171</span> * &lt;p&gt;&lt;b&gt;Cluster Configuration&lt;/b&gt;&lt;/p&gt;<a name="line.171"></a>
<span class="sourceLineNo">172</span> *<a name="line.172"></a>
<span class="sourceLineNo">173</span> * &lt;p&gt;<a name="line.173"></a>
<span class="sourceLineNo">174</span> * Instances support dynamic cluster configuration changes at runtime. Initially, all nodes are in an &lt;i&gt;unconfigured&lt;/i&gt;<a name="line.174"></a>
<span class="sourceLineNo">175</span> * state, where nothing has been added to the Raft log yet and no cluster is defined. Unconfigured nodes are passive: they<a name="line.175"></a>
<span class="sourceLineNo">176</span> * stay in follower mode (i.e., they will not start elections), and they disallow local transactions that make any changes<a name="line.176"></a>
<span class="sourceLineNo">177</span> * (other than as described below to initialize a new cluster).<a name="line.177"></a>
<span class="sourceLineNo">178</span> *<a name="line.178"></a>
<span class="sourceLineNo">179</span> * &lt;p&gt;<a name="line.179"></a>
<span class="sourceLineNo">180</span> * A node is configured if and only if it has a &lt;i&gt;configuration&lt;/i&gt;. A cluster configuration consists of the identites of<a name="line.180"></a>
<span class="sourceLineNo">181</span> * the nodes in the cluster, and their corresponding network addresses. Once a node &lt;i&gt;configured&lt;/i&gt;, a separate issue<a name="line.181"></a>
<span class="sourceLineNo">182</span> * is whether a node is &lt;i&gt;included&lt;/i&gt; in its own configuration. A node that is not included in its own configuration<a name="line.182"></a>
<span class="sourceLineNo">183</span> * does not count its own vote for elections and log commits, but otherwise functions normally in the cluster.<a name="line.183"></a>
<span class="sourceLineNo">184</span> *<a name="line.184"></a>
<span class="sourceLineNo">185</span> * &lt;p&gt;<a name="line.185"></a>
<span class="sourceLineNo">186</span> * An unconfigured node becomes configured when either:<a name="line.186"></a>
<span class="sourceLineNo">187</span> * &lt;ul&gt;<a name="line.187"></a>
<span class="sourceLineNo">188</span> *  &lt;li&gt;An {@link AppendRequest} is received from a leader of some existing cluster, in which case the node<a name="line.188"></a>
<span class="sourceLineNo">189</span> *      joins the cluster and applies the received cluster configuration; or&lt;/li&gt;<a name="line.189"></a>
<span class="sourceLineNo">190</span> *  &lt;li&gt;{@link RaftKVTransaction#configChange RaftKVTransaction.configChange()} is invoked and committed within<a name="line.190"></a>
<span class="sourceLineNo">191</span> *      a local transaction, which creates a new cluster and configures it to contain the local node&lt;/li&gt;<a name="line.191"></a>
<span class="sourceLineNo">192</span> * &lt;/ul&gt;<a name="line.192"></a>
<span class="sourceLineNo">193</span> *<a name="line.193"></a>
<span class="sourceLineNo">194</span> * &lt;p&gt;<a name="line.194"></a>
<span class="sourceLineNo">195</span> * On the first commit, newly created clusters are assigned a random 32-bit cluster ID. This ID is included in all<a name="line.195"></a>
<span class="sourceLineNo">196</span> * messages sent over the network, and adopted by unconfigured nodes that join the cluster. Configured nodes will discard<a name="line.196"></a>
<span class="sourceLineNo">197</span> * incoming messages containing a different cluster ID. This prevents data corruption that can occur if nodes from two<a name="line.197"></a>
<span class="sourceLineNo">198</span> * different clusters are inadvertently "mixed" together.<a name="line.198"></a>
<span class="sourceLineNo">199</span> *<a name="line.199"></a>
<span class="sourceLineNo">200</span> * &lt;p&gt;<a name="line.200"></a>
<span class="sourceLineNo">201</span> * Once a node joins a cluster (identified by cluster ID), it cannot be reassigned to a different cluster without first<a name="line.201"></a>
<span class="sourceLineNo">202</span> * returning it to the unconfigured state; to do that, it must be shut it down and its persistent state directory deleted.<a name="line.202"></a>
<span class="sourceLineNo">203</span> *<a name="line.203"></a>
<span class="sourceLineNo">204</span> * @see &lt;a href="https://raftconsensus.github.io/"&gt;The Raft Consensus Algorithm&lt;/a&gt;<a name="line.204"></a>
<span class="sourceLineNo">205</span> */<a name="line.205"></a>
<span class="sourceLineNo">206</span>public class RaftKVDatabase implements KVDatabase {<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
<span class="sourceLineNo">208</span>    /**<a name="line.208"></a>
<span class="sourceLineNo">209</span>     * Default minimum election timeout ({@value #DEFAULT_MIN_ELECTION_TIMEOUT}ms).<a name="line.209"></a>
<span class="sourceLineNo">210</span>     *<a name="line.210"></a>
<span class="sourceLineNo">211</span>     * @see #setMinElectionTimeout<a name="line.211"></a>
<span class="sourceLineNo">212</span>     */<a name="line.212"></a>
<span class="sourceLineNo">213</span>    public static final int DEFAULT_MIN_ELECTION_TIMEOUT = 750;<a name="line.213"></a>
<span class="sourceLineNo">214</span><a name="line.214"></a>
<span class="sourceLineNo">215</span>    /**<a name="line.215"></a>
<span class="sourceLineNo">216</span>     * Default maximum election timeout ({@value #DEFAULT_MAX_ELECTION_TIMEOUT}ms).<a name="line.216"></a>
<span class="sourceLineNo">217</span>     *<a name="line.217"></a>
<span class="sourceLineNo">218</span>     * @see #setMaxElectionTimeout<a name="line.218"></a>
<span class="sourceLineNo">219</span>     */<a name="line.219"></a>
<span class="sourceLineNo">220</span>    public static final int DEFAULT_MAX_ELECTION_TIMEOUT = 1000;<a name="line.220"></a>
<span class="sourceLineNo">221</span><a name="line.221"></a>
<span class="sourceLineNo">222</span>    /**<a name="line.222"></a>
<span class="sourceLineNo">223</span>     * Default heartbeat timeout ({@value DEFAULT_HEARTBEAT_TIMEOUT}ms).<a name="line.223"></a>
<span class="sourceLineNo">224</span>     *<a name="line.224"></a>
<span class="sourceLineNo">225</span>     * @see #setHeartbeatTimeout<a name="line.225"></a>
<span class="sourceLineNo">226</span>     */<a name="line.226"></a>
<span class="sourceLineNo">227</span>    public static final int DEFAULT_HEARTBEAT_TIMEOUT = 200;<a name="line.227"></a>
<span class="sourceLineNo">228</span><a name="line.228"></a>
<span class="sourceLineNo">229</span>    /**<a name="line.229"></a>
<span class="sourceLineNo">230</span>     * Default maximum supported outstanding transaction duration ({@value DEFAULT_MAX_TRANSACTION_DURATION}ms).<a name="line.230"></a>
<span class="sourceLineNo">231</span>     *<a name="line.231"></a>
<span class="sourceLineNo">232</span>     * @see #setMaxTransactionDuration<a name="line.232"></a>
<span class="sourceLineNo">233</span>     */<a name="line.233"></a>
<span class="sourceLineNo">234</span>    public static final int DEFAULT_MAX_TRANSACTION_DURATION = 10 * 1000;<a name="line.234"></a>
<span class="sourceLineNo">235</span><a name="line.235"></a>
<span class="sourceLineNo">236</span>    /**<a name="line.236"></a>
<span class="sourceLineNo">237</span>     * Default maximum supported applied log entry memory usage ({@value DEFAULT_MAX_APPLIED_LOG_MEMORY} bytes).<a name="line.237"></a>
<span class="sourceLineNo">238</span>     *<a name="line.238"></a>
<span class="sourceLineNo">239</span>     * @see #setMaxAppliedLogMemory<a name="line.239"></a>
<span class="sourceLineNo">240</span>     */<a name="line.240"></a>
<span class="sourceLineNo">241</span>    public static final long DEFAULT_MAX_APPLIED_LOG_MEMORY = 100 * 1024 * 1024;                // 100MB<a name="line.241"></a>
<span class="sourceLineNo">242</span><a name="line.242"></a>
<span class="sourceLineNo">243</span>    /**<a name="line.243"></a>
<span class="sourceLineNo">244</span>     * Default transaction commit timeout ({@value DEFAULT_COMMIT_TIMEOUT}).<a name="line.244"></a>
<span class="sourceLineNo">245</span>     *<a name="line.245"></a>
<span class="sourceLineNo">246</span>     * @see #setCommitTimeout<a name="line.246"></a>
<span class="sourceLineNo">247</span>     * @see RaftKVTransaction#setTimeout<a name="line.247"></a>
<span class="sourceLineNo">248</span>     */<a name="line.248"></a>
<span class="sourceLineNo">249</span>    public static final int DEFAULT_COMMIT_TIMEOUT = 5000;                                      // 5 seconds<a name="line.249"></a>
<span class="sourceLineNo">250</span><a name="line.250"></a>
<span class="sourceLineNo">251</span>    // Internal constants<a name="line.251"></a>
<span class="sourceLineNo">252</span>    private static final int MAX_SNAPSHOT_TRANSMIT_AGE = (int)TimeUnit.SECONDS.toMillis(90);    // 90 seconds<a name="line.252"></a>
<span class="sourceLineNo">253</span>    private static final int MAX_SLOW_FOLLOWER_APPLY_DELAY_HEARTBEATS = 10;<a name="line.253"></a>
<span class="sourceLineNo">254</span>    private static final int MAX_APPLIED_LOG_ENTRIES = 32;<a name="line.254"></a>
<span class="sourceLineNo">255</span>    private static final int FOLLOWER_LINGER_HEARTBEATS = 3;                // how long to keep updating removed followers<a name="line.255"></a>
<span class="sourceLineNo">256</span>    private static final float MAX_CLOCK_DRIFT = 0.01f;                     // max clock drift (ratio) in one min election timeout<a name="line.256"></a>
<span class="sourceLineNo">257</span><a name="line.257"></a>
<span class="sourceLineNo">258</span>    // File prefixes and suffixes<a name="line.258"></a>
<span class="sourceLineNo">259</span>    private static final String TX_FILE_PREFIX = "tx-";<a name="line.259"></a>
<span class="sourceLineNo">260</span>    private static final String TEMP_FILE_PREFIX = "temp-";<a name="line.260"></a>
<span class="sourceLineNo">261</span>    private static final String TEMP_FILE_SUFFIX = ".tmp";<a name="line.261"></a>
<span class="sourceLineNo">262</span>    private static final String KVSTORE_FILE_SUFFIX = ".kvstore";<a name="line.262"></a>
<span class="sourceLineNo">263</span>    private static final Pattern TEMP_FILE_PATTERN = Pattern.compile(".*" + Pattern.quote(TEMP_FILE_SUFFIX));<a name="line.263"></a>
<span class="sourceLineNo">264</span>    private static final Pattern KVSTORE_FILE_PATTERN = Pattern.compile(".*" + Pattern.quote(KVSTORE_FILE_SUFFIX));<a name="line.264"></a>
<span class="sourceLineNo">265</span><a name="line.265"></a>
<span class="sourceLineNo">266</span>    // Keys for persistent Raft state<a name="line.266"></a>
<span class="sourceLineNo">267</span>    private static final byte[] CLUSTER_ID_KEY = ByteUtil.parse("0001");<a name="line.267"></a>
<span class="sourceLineNo">268</span>    private static final byte[] CURRENT_TERM_KEY = ByteUtil.parse("0002");<a name="line.268"></a>
<span class="sourceLineNo">269</span>    private static final byte[] LAST_APPLIED_TERM_KEY = ByteUtil.parse("0003");<a name="line.269"></a>
<span class="sourceLineNo">270</span>    private static final byte[] LAST_APPLIED_INDEX_KEY = ByteUtil.parse("0004");<a name="line.270"></a>
<span class="sourceLineNo">271</span>    private static final byte[] LAST_APPLIED_CONFIG_KEY = ByteUtil.parse("0005");<a name="line.271"></a>
<span class="sourceLineNo">272</span>    private static final byte[] VOTED_FOR_KEY = ByteUtil.parse("0006");<a name="line.272"></a>
<span class="sourceLineNo">273</span><a name="line.273"></a>
<span class="sourceLineNo">274</span>    // Prefix for all state machine key/value keys<a name="line.274"></a>
<span class="sourceLineNo">275</span>    private static final byte[] STATE_MACHINE_PREFIX = ByteUtil.parse("80");<a name="line.275"></a>
<span class="sourceLineNo">276</span><a name="line.276"></a>
<span class="sourceLineNo">277</span>    // Logging<a name="line.277"></a>
<span class="sourceLineNo">278</span>    private final Logger log = LoggerFactory.getLogger(this.getClass());<a name="line.278"></a>
<span class="sourceLineNo">279</span><a name="line.279"></a>
<span class="sourceLineNo">280</span>    // Configuration state<a name="line.280"></a>
<span class="sourceLineNo">281</span>    private Network network = new TCPNetwork();<a name="line.281"></a>
<span class="sourceLineNo">282</span>    private String identity;<a name="line.282"></a>
<span class="sourceLineNo">283</span>    private AtomicKVStore kvstore;<a name="line.283"></a>
<span class="sourceLineNo">284</span>    private int minElectionTimeout = DEFAULT_MIN_ELECTION_TIMEOUT;<a name="line.284"></a>
<span class="sourceLineNo">285</span>    private int maxElectionTimeout = DEFAULT_MAX_ELECTION_TIMEOUT;<a name="line.285"></a>
<span class="sourceLineNo">286</span>    private int heartbeatTimeout = DEFAULT_HEARTBEAT_TIMEOUT;<a name="line.286"></a>
<span class="sourceLineNo">287</span>    private int maxTransactionDuration = DEFAULT_MAX_TRANSACTION_DURATION;<a name="line.287"></a>
<span class="sourceLineNo">288</span>    private int commitTimeout = DEFAULT_COMMIT_TIMEOUT;<a name="line.288"></a>
<span class="sourceLineNo">289</span>    private long maxAppliedLogMemory = DEFAULT_MAX_APPLIED_LOG_MEMORY;<a name="line.289"></a>
<span class="sourceLineNo">290</span>    private File logDir;<a name="line.290"></a>
<span class="sourceLineNo">291</span><a name="line.291"></a>
<span class="sourceLineNo">292</span>    // Raft runtime state<a name="line.292"></a>
<span class="sourceLineNo">293</span>    private Role role;                                                  // Raft state: LEADER, FOLLOWER, or CANDIDATE<a name="line.293"></a>
<span class="sourceLineNo">294</span>    private SecureRandom random;                                        // used to randomize election timeout, etc.<a name="line.294"></a>
<span class="sourceLineNo">295</span>    private int clusterId;                                              // cluster ID (zero if unconfigured - usually)<a name="line.295"></a>
<span class="sourceLineNo">296</span>    private long currentTerm;                                           // current Raft term (zero if unconfigured)<a name="line.296"></a>
<span class="sourceLineNo">297</span>    private long commitIndex;                                           // current Raft commit index (zero if unconfigured)<a name="line.297"></a>
<span class="sourceLineNo">298</span>    private long lastAppliedTerm;                                       // key/value store last applied term (zero if unconfigured)<a name="line.298"></a>
<span class="sourceLineNo">299</span>    private long lastAppliedIndex;                                      // key/value store last applied index (zero if unconfigured)<a name="line.299"></a>
<span class="sourceLineNo">300</span>    private final ArrayList&lt;LogEntry&gt; raftLog = new ArrayList&lt;&gt;();      // unapplied log entries (empty if unconfigured)<a name="line.300"></a>
<span class="sourceLineNo">301</span>    private Map&lt;String, String&gt; lastAppliedConfig;                      // key/value store last applied config (empty if none)<a name="line.301"></a>
<span class="sourceLineNo">302</span>    private Map&lt;String, String&gt; currentConfig;                          // most recent cluster config (empty if unconfigured)<a name="line.302"></a>
<span class="sourceLineNo">303</span><a name="line.303"></a>
<span class="sourceLineNo">304</span>    // Non-Raft runtime state<a name="line.304"></a>
<span class="sourceLineNo">305</span>    private AtomicKVStore kv;<a name="line.305"></a>
<span class="sourceLineNo">306</span>    private FileChannel logDirChannel;<a name="line.306"></a>
<span class="sourceLineNo">307</span>    private ScheduledExecutorService executor;<a name="line.307"></a>
<span class="sourceLineNo">308</span>    private String returnAddress;                                       // return address for message currently being processed<a name="line.308"></a>
<span class="sourceLineNo">309</span>    private final HashSet&lt;String&gt; transmitting = new HashSet&lt;&gt;();       // network addresses whose output queues are not empty<a name="line.309"></a>
<span class="sourceLineNo">310</span>    private final HashMap&lt;Long, RaftKVTransaction&gt; openTransactions = new HashMap&lt;&gt;();<a name="line.310"></a>
<span class="sourceLineNo">311</span>    private final LinkedHashSet&lt;Service&gt; pendingService = new LinkedHashSet&lt;&gt;();<a name="line.311"></a>
<span class="sourceLineNo">312</span>    private boolean performingService;<a name="line.312"></a>
<span class="sourceLineNo">313</span>    private boolean shuttingDown;                                       // prevents new transactions from being created<a name="line.313"></a>
<span class="sourceLineNo">314</span><a name="line.314"></a>
<span class="sourceLineNo">315</span>// Configuration<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
<span class="sourceLineNo">317</span>    /**<a name="line.317"></a>
<span class="sourceLineNo">318</span>     * Configure the {@link AtomicKVStore} in which local persistent state is stored.<a name="line.318"></a>
<span class="sourceLineNo">319</span>     *<a name="line.319"></a>
<span class="sourceLineNo">320</span>     * &lt;p&gt;<a name="line.320"></a>
<span class="sourceLineNo">321</span>     * If this property is left unconfigured, by default a {@link LevelDBKVStore} is created when<a name="line.321"></a>
<span class="sourceLineNo">322</span>     * {@link #start} is invoked and shutdown on {@link #stop}.<a name="line.322"></a>
<span class="sourceLineNo">323</span>     *<a name="line.323"></a>
<span class="sourceLineNo">324</span>     * @param kvstore local persistent data store<a name="line.324"></a>
<span class="sourceLineNo">325</span>     * @throws IllegalStateException if this instance is already started<a name="line.325"></a>
<span class="sourceLineNo">326</span>     */<a name="line.326"></a>
<span class="sourceLineNo">327</span>    public synchronized void setKVStore(AtomicKVStore kvstore) {<a name="line.327"></a>
<span class="sourceLineNo">328</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.328"></a>
<span class="sourceLineNo">329</span>        this.kvstore = kvstore;<a name="line.329"></a>
<span class="sourceLineNo">330</span>    }<a name="line.330"></a>
<span class="sourceLineNo">331</span><a name="line.331"></a>
<span class="sourceLineNo">332</span>    /**<a name="line.332"></a>
<span class="sourceLineNo">333</span>     * Configure the directory in which uncommitted log entries are stored.<a name="line.333"></a>
<span class="sourceLineNo">334</span>     *<a name="line.334"></a>
<span class="sourceLineNo">335</span>     * &lt;p&gt;<a name="line.335"></a>
<span class="sourceLineNo">336</span>     * Required property.<a name="line.336"></a>
<span class="sourceLineNo">337</span>     *<a name="line.337"></a>
<span class="sourceLineNo">338</span>     * @param directory log directory<a name="line.338"></a>
<span class="sourceLineNo">339</span>     * @throws IllegalStateException if this instance is already started<a name="line.339"></a>
<span class="sourceLineNo">340</span>     */<a name="line.340"></a>
<span class="sourceLineNo">341</span>    public synchronized void setLogDirectory(File directory) {<a name="line.341"></a>
<span class="sourceLineNo">342</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.342"></a>
<span class="sourceLineNo">343</span>        this.logDir = directory;<a name="line.343"></a>
<span class="sourceLineNo">344</span>    }<a name="line.344"></a>
<span class="sourceLineNo">345</span><a name="line.345"></a>
<span class="sourceLineNo">346</span>    /**<a name="line.346"></a>
<span class="sourceLineNo">347</span>     * Configure the {@link Network} to use for inter-node communication.<a name="line.347"></a>
<span class="sourceLineNo">348</span>     *<a name="line.348"></a>
<span class="sourceLineNo">349</span>     * &lt;p&gt;<a name="line.349"></a>
<span class="sourceLineNo">350</span>     * By default, a {@link TCPNetwork} instance is used.<a name="line.350"></a>
<span class="sourceLineNo">351</span>     *<a name="line.351"></a>
<span class="sourceLineNo">352</span>     * @param network network implementation; must not be {@linkplain Network#start started}<a name="line.352"></a>
<span class="sourceLineNo">353</span>     * @throws IllegalStateException if this instance is already started<a name="line.353"></a>
<span class="sourceLineNo">354</span>     */<a name="line.354"></a>
<span class="sourceLineNo">355</span>    public synchronized void setNetwork(Network network) {<a name="line.355"></a>
<span class="sourceLineNo">356</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.356"></a>
<span class="sourceLineNo">357</span>        this.network = network;<a name="line.357"></a>
<span class="sourceLineNo">358</span>    }<a name="line.358"></a>
<span class="sourceLineNo">359</span><a name="line.359"></a>
<span class="sourceLineNo">360</span>    /**<a name="line.360"></a>
<span class="sourceLineNo">361</span>     * Configure the Raft identity.<a name="line.361"></a>
<span class="sourceLineNo">362</span>     *<a name="line.362"></a>
<span class="sourceLineNo">363</span>     * &lt;p&gt;<a name="line.363"></a>
<span class="sourceLineNo">364</span>     * Required property unless this is a single-node cluster.<a name="line.364"></a>
<span class="sourceLineNo">365</span>     *<a name="line.365"></a>
<span class="sourceLineNo">366</span>     * @param identity unique Raft identity of this node in its cluster<a name="line.366"></a>
<span class="sourceLineNo">367</span>     * @throws IllegalStateException if this instance is already started<a name="line.367"></a>
<span class="sourceLineNo">368</span>     */<a name="line.368"></a>
<span class="sourceLineNo">369</span>    public synchronized void setIdentity(String identity) {<a name="line.369"></a>
<span class="sourceLineNo">370</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.370"></a>
<span class="sourceLineNo">371</span>        this.identity = identity;<a name="line.371"></a>
<span class="sourceLineNo">372</span>    }<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
<span class="sourceLineNo">374</span>    /**<a name="line.374"></a>
<span class="sourceLineNo">375</span>     * Get this node's Raft identity.<a name="line.375"></a>
<span class="sourceLineNo">376</span>     *<a name="line.376"></a>
<span class="sourceLineNo">377</span>     * @return the unique identity of this node in its cluster<a name="line.377"></a>
<span class="sourceLineNo">378</span>     */<a name="line.378"></a>
<span class="sourceLineNo">379</span>    public synchronized String getIdentity() {<a name="line.379"></a>
<span class="sourceLineNo">380</span>        return this.identity;<a name="line.380"></a>
<span class="sourceLineNo">381</span>    }<a name="line.381"></a>
<span class="sourceLineNo">382</span><a name="line.382"></a>
<span class="sourceLineNo">383</span>    /**<a name="line.383"></a>
<span class="sourceLineNo">384</span>     * Configure the minimum election timeout.<a name="line.384"></a>
<span class="sourceLineNo">385</span>     *<a name="line.385"></a>
<span class="sourceLineNo">386</span>     * &lt;p&gt;<a name="line.386"></a>
<span class="sourceLineNo">387</span>     * This must be set to a value greater than the {@linkplain #setHeartbeatTimeout heartbeat timeout}.<a name="line.387"></a>
<span class="sourceLineNo">388</span>     *<a name="line.388"></a>
<span class="sourceLineNo">389</span>     * &lt;p&gt;<a name="line.389"></a>
<span class="sourceLineNo">390</span>     * Default is {@link #DEFAULT_MIN_ELECTION_TIMEOUT}.<a name="line.390"></a>
<span class="sourceLineNo">391</span>     *<a name="line.391"></a>
<span class="sourceLineNo">392</span>     * &lt;p&gt;<a name="line.392"></a>
<span class="sourceLineNo">393</span>     * &lt;b&gt;Warning:&lt;/b&gt; currently all nodes must have the same configured minimum election timeout,<a name="line.393"></a>
<span class="sourceLineNo">394</span>     * otherwise read-only transactions are not guaranteed to be completely up-to-date.<a name="line.394"></a>
<span class="sourceLineNo">395</span>     *<a name="line.395"></a>
<span class="sourceLineNo">396</span>     * @param timeout minimum election timeout in milliseconds<a name="line.396"></a>
<span class="sourceLineNo">397</span>     * @throws IllegalStateException if this instance is already started<a name="line.397"></a>
<span class="sourceLineNo">398</span>     * @throws IllegalArgumentException if {@code timeout &lt;= 0}<a name="line.398"></a>
<span class="sourceLineNo">399</span>     */<a name="line.399"></a>
<span class="sourceLineNo">400</span>    public synchronized void setMinElectionTimeout(int timeout) {<a name="line.400"></a>
<span class="sourceLineNo">401</span>        Preconditions.checkArgument(timeout &gt; 0, "timeout &lt;= 0");<a name="line.401"></a>
<span class="sourceLineNo">402</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.402"></a>
<span class="sourceLineNo">403</span>        this.minElectionTimeout = timeout;<a name="line.403"></a>
<span class="sourceLineNo">404</span>    }<a name="line.404"></a>
<span class="sourceLineNo">405</span><a name="line.405"></a>
<span class="sourceLineNo">406</span>    /**<a name="line.406"></a>
<span class="sourceLineNo">407</span>     * Configure the maximum election timeout.<a name="line.407"></a>
<span class="sourceLineNo">408</span>     *<a name="line.408"></a>
<span class="sourceLineNo">409</span>     * &lt;p&gt;<a name="line.409"></a>
<span class="sourceLineNo">410</span>     * Default is {@link #DEFAULT_MAX_ELECTION_TIMEOUT}.<a name="line.410"></a>
<span class="sourceLineNo">411</span>     *<a name="line.411"></a>
<span class="sourceLineNo">412</span>     * @param timeout maximum election timeout in milliseconds<a name="line.412"></a>
<span class="sourceLineNo">413</span>     * @throws IllegalStateException if this instance is already started<a name="line.413"></a>
<span class="sourceLineNo">414</span>     * @throws IllegalArgumentException if {@code timeout &lt;= 0}<a name="line.414"></a>
<span class="sourceLineNo">415</span>     */<a name="line.415"></a>
<span class="sourceLineNo">416</span>    public synchronized void setMaxElectionTimeout(int timeout) {<a name="line.416"></a>
<span class="sourceLineNo">417</span>        Preconditions.checkArgument(timeout &gt; 0, "timeout &lt;= 0");<a name="line.417"></a>
<span class="sourceLineNo">418</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.418"></a>
<span class="sourceLineNo">419</span>        this.maxElectionTimeout = timeout;<a name="line.419"></a>
<span class="sourceLineNo">420</span>    }<a name="line.420"></a>
<span class="sourceLineNo">421</span><a name="line.421"></a>
<span class="sourceLineNo">422</span>    /**<a name="line.422"></a>
<span class="sourceLineNo">423</span>     * Configure the heartbeat timeout.<a name="line.423"></a>
<span class="sourceLineNo">424</span>     *<a name="line.424"></a>
<span class="sourceLineNo">425</span>     * &lt;p&gt;<a name="line.425"></a>
<span class="sourceLineNo">426</span>     * This must be set to a value less than the {@linkplain #setMinElectionTimeout minimum election timeout}.<a name="line.426"></a>
<span class="sourceLineNo">427</span>     *<a name="line.427"></a>
<span class="sourceLineNo">428</span>     * &lt;p&gt;<a name="line.428"></a>
<span class="sourceLineNo">429</span>     * Default is {@link #DEFAULT_HEARTBEAT_TIMEOUT}.<a name="line.429"></a>
<span class="sourceLineNo">430</span>     *<a name="line.430"></a>
<span class="sourceLineNo">431</span>     * @param timeout heartbeat timeout in milliseconds<a name="line.431"></a>
<span class="sourceLineNo">432</span>     * @throws IllegalStateException if this instance is already started<a name="line.432"></a>
<span class="sourceLineNo">433</span>     * @throws IllegalArgumentException if {@code timeout &lt;= 0}<a name="line.433"></a>
<span class="sourceLineNo">434</span>     */<a name="line.434"></a>
<span class="sourceLineNo">435</span>    public synchronized void setHeartbeatTimeout(int timeout) {<a name="line.435"></a>
<span class="sourceLineNo">436</span>        Preconditions.checkArgument(timeout &gt; 0, "timeout &lt;= 0");<a name="line.436"></a>
<span class="sourceLineNo">437</span>        Preconditions.checkState(this.role == null, "already started");<a name="line.437"></a>
<span class="sourceLineNo">438</span>        this.heartbeatTimeout = timeout;<a name="line.438"></a>
<span class="sourceLineNo">439</span>    }<a name="line.439"></a>
<span class="sourceLineNo">440</span><a name="line.440"></a>
<span class="sourceLineNo">441</span>    /**<a name="line.441"></a>
<span class="sourceLineNo">442</span>     * Configure the maximum supported duration for outstanding transactions.<a name="line.442"></a>
<span class="sourceLineNo">443</span>     *<a name="line.443"></a>
<span class="sourceLineNo">444</span>     * &lt;p&gt;<a name="line.444"></a>
<span class="sourceLineNo">445</span>     * This value is the &lt;i&gt;T&lt;sub&gt;max&lt;/sub&gt;&lt;/i&gt; value from the {@linkplain RaftKVDatabase overview}.<a name="line.445"></a>
<span class="sourceLineNo">446</span>     * A larger value means more memory may be used.<a name="line.446"></a>
<span class="sourceLineNo">447</span>     *<a name="line.447"></a>
<span class="sourceLineNo">448</span>     * &lt;p&gt;<a name="line.448"></a>
<span class="sourceLineNo">449</span>     * This value may be changed while this instance is already running.<a name="line.449"></a>
<span class="sourceLineNo">450</span>     *<a name="line.450"></a>
<span class="sourceLineNo">451</span>     * &lt;p&gt;<a name="line.451"></a>
<span class="sourceLineNo">452</span>     * Default is {@link #DEFAULT_MAX_TRANSACTION_DURATION}.<a name="line.452"></a>
<span class="sourceLineNo">453</span>     *<a name="line.453"></a>
<span class="sourceLineNo">454</span>     * @param duration maximum supported duration for outstanding transactions in milliseconds<a name="line.454"></a>
<span class="sourceLineNo">455</span>     * @throws IllegalArgumentException if {@code duration &lt;= 0}<a name="line.455"></a>
<span class="sourceLineNo">456</span>     * @see #setMaxAppliedLogMemory<a name="line.456"></a>
<span class="sourceLineNo">457</span>     */<a name="line.457"></a>
<span class="sourceLineNo">458</span>    public synchronized void setMaxTransactionDuration(int duration) {<a name="line.458"></a>
<span class="sourceLineNo">459</span>        Preconditions.checkArgument(duration &gt; 0, "duration &lt;= 0");<a name="line.459"></a>
<span class="sourceLineNo">460</span>        this.maxTransactionDuration = duration;<a name="line.460"></a>
<span class="sourceLineNo">461</span>    }<a name="line.461"></a>
<span class="sourceLineNo">462</span><a name="line.462"></a>
<span class="sourceLineNo">463</span>    /**<a name="line.463"></a>
<span class="sourceLineNo">464</span>     * Configure the maximum allowed memory used for caching log entries that have already been applied to the state machine.<a name="line.464"></a>
<span class="sourceLineNo">465</span>     *<a name="line.465"></a>
<span class="sourceLineNo">466</span>     * &lt;p&gt;<a name="line.466"></a>
<span class="sourceLineNo">467</span>     * This value is the &lt;i&gt;M&lt;sub&gt;max&lt;/sub&gt;&lt;/i&gt; value from the {@linkplain RaftKVDatabase overview}.<a name="line.467"></a>
<span class="sourceLineNo">468</span>     * A higher value means transactions may be larger and/or stay open longer without causing a {@link RetryTransactionException}.<a name="line.468"></a>
<span class="sourceLineNo">469</span>     *<a name="line.469"></a>
<span class="sourceLineNo">470</span>     * &lt;p&gt;<a name="line.470"></a>
<span class="sourceLineNo">471</span>     * This value is approximate, and only affects leaders; followers do not cache applied log entries.<a name="line.471"></a>
<span class="sourceLineNo">472</span>     *<a name="line.472"></a>
<span class="sourceLineNo">473</span>     * &lt;p&gt;<a name="line.473"></a>
<span class="sourceLineNo">474</span>     * This value may be changed while this instance is already running.<a name="line.474"></a>
<span class="sourceLineNo">475</span>     *<a name="line.475"></a>
<span class="sourceLineNo">476</span>     * &lt;p&gt;<a name="line.476"></a>
<span class="sourceLineNo">477</span>     * Default is {@link #DEFAULT_MAX_APPLIED_LOG_MEMORY}.<a name="line.477"></a>
<span class="sourceLineNo">478</span>     *<a name="line.478"></a>
<span class="sourceLineNo">479</span>     * @param memory maximum allowed memory usage for cached applied log entries<a name="line.479"></a>
<span class="sourceLineNo">480</span>     * @throws IllegalArgumentException if {@code memory &lt;= 0}<a name="line.480"></a>
<span class="sourceLineNo">481</span>     * @see #setMaxTransactionDuration<a name="line.481"></a>
<span class="sourceLineNo">482</span>     */<a name="line.482"></a>
<span class="sourceLineNo">483</span>    public synchronized void setMaxAppliedLogMemory(long memory) {<a name="line.483"></a>
<span class="sourceLineNo">484</span>        Preconditions.checkArgument(memory &gt; 0, "memory &lt;= 0");<a name="line.484"></a>
<span class="sourceLineNo">485</span>        this.maxAppliedLogMemory = memory;<a name="line.485"></a>
<span class="sourceLineNo">486</span>    }<a name="line.486"></a>
<span class="sourceLineNo">487</span><a name="line.487"></a>
<span class="sourceLineNo">488</span>    /**<a name="line.488"></a>
<span class="sourceLineNo">489</span>     * Configure the default transaction commit timeout.<a name="line.489"></a>
<span class="sourceLineNo">490</span>     *<a name="line.490"></a>
<span class="sourceLineNo">491</span>     * &lt;p&gt;<a name="line.491"></a>
<span class="sourceLineNo">492</span>     * This value determines how transactions will wait once {@link RaftKVTransaction#commit commit()}<a name="line.492"></a>
<span class="sourceLineNo">493</span>     * is invoked for the commit to succeed. This can be overridden on a per-transaction basis via<a name="line.493"></a>
<span class="sourceLineNo">494</span>     * {@link RaftKVTransaction#setTimeout}.<a name="line.494"></a>
<span class="sourceLineNo">495</span>     *<a name="line.495"></a>
<span class="sourceLineNo">496</span>     * &lt;p&gt;<a name="line.496"></a>
<span class="sourceLineNo">497</span>     * This value may be changed while this instance is already running.<a name="line.497"></a>
<span class="sourceLineNo">498</span>     *<a name="line.498"></a>
<span class="sourceLineNo">499</span>     * &lt;p&gt;<a name="line.499"></a>
<span class="sourceLineNo">500</span>     * Default is {@link #DEFAULT_COMMIT_TIMEOUT}.<a name="line.500"></a>
<span class="sourceLineNo">501</span>     *<a name="line.501"></a>
<span class="sourceLineNo">502</span>     * @param timeout transaction commit timeout in milliseconds, or zero for unlimited<a name="line.502"></a>
<span class="sourceLineNo">503</span>     * @throws IllegalArgumentException if {@code timeout} is negative<a name="line.503"></a>
<span class="sourceLineNo">504</span>     * @see RaftKVTransaction#setTimeout<a name="line.504"></a>
<span class="sourceLineNo">505</span>     */<a name="line.505"></a>
<span class="sourceLineNo">506</span>    public synchronized void setCommitTimeout(int timeout) {<a name="line.506"></a>
<span class="sourceLineNo">507</span>        Preconditions.checkArgument(timeout &gt;= 0, "timeout &lt; 0");<a name="line.507"></a>
<span class="sourceLineNo">508</span>        this.commitTimeout = commitTimeout;<a name="line.508"></a>
<span class="sourceLineNo">509</span>    }<a name="line.509"></a>
<span class="sourceLineNo">510</span><a name="line.510"></a>
<span class="sourceLineNo">511</span>// Lifecycle<a name="line.511"></a>
<span class="sourceLineNo">512</span><a name="line.512"></a>
<span class="sourceLineNo">513</span>    /**<a name="line.513"></a>
<span class="sourceLineNo">514</span>     * Start this instance.<a name="line.514"></a>
<span class="sourceLineNo">515</span>     *<a name="line.515"></a>
<span class="sourceLineNo">516</span>     * &lt;p&gt;<a name="line.516"></a>
<span class="sourceLineNo">517</span>     * Does nothing if already started or in the process of shutting down.<a name="line.517"></a>
<span class="sourceLineNo">518</span>     * &lt;/p&gt;<a name="line.518"></a>
<span class="sourceLineNo">519</span>     *<a name="line.519"></a>
<span class="sourceLineNo">520</span>     * @throws IllegalStateException if this instance is not properly configured<a name="line.520"></a>
<span class="sourceLineNo">521</span>     * @throws IllegalArgumentException if persistent data is corrupted<a name="line.521"></a>
<span class="sourceLineNo">522</span>     * @throws IOException if an I/O error occurs<a name="line.522"></a>
<span class="sourceLineNo">523</span>     */<a name="line.523"></a>
<span class="sourceLineNo">524</span>    @PostConstruct<a name="line.524"></a>
<span class="sourceLineNo">525</span>    public synchronized void start() throws IOException {<a name="line.525"></a>
<span class="sourceLineNo">526</span><a name="line.526"></a>
<span class="sourceLineNo">527</span>        // Sanity check<a name="line.527"></a>
<span class="sourceLineNo">528</span>        assert this.checkState();<a name="line.528"></a>
<span class="sourceLineNo">529</span>        if (this.role != null)<a name="line.529"></a>
<span class="sourceLineNo">530</span>            return;<a name="line.530"></a>
<span class="sourceLineNo">531</span>        Preconditions.checkState(!this.shuttingDown, "shutdown in progress");<a name="line.531"></a>
<span class="sourceLineNo">532</span>        Preconditions.checkState(this.logDir != null, "no log directory configured");<a name="line.532"></a>
<span class="sourceLineNo">533</span>        Preconditions.checkState(this.network != null, "no network configured");<a name="line.533"></a>
<span class="sourceLineNo">534</span>        Preconditions.checkState(this.kv == null, "key/value store exists");<a name="line.534"></a>
<span class="sourceLineNo">535</span>        Preconditions.checkState(this.minElectionTimeout &lt;= this.maxElectionTimeout, "minElectionTimeout &gt; maxElectionTimeout");<a name="line.535"></a>
<span class="sourceLineNo">536</span>        Preconditions.checkState(this.heartbeatTimeout &lt; this.minElectionTimeout, "heartbeatTimeout &gt;= minElectionTimeout");<a name="line.536"></a>
<span class="sourceLineNo">537</span>        Preconditions.checkState(this.identity != null, "no identity configured");<a name="line.537"></a>
<span class="sourceLineNo">538</span><a name="line.538"></a>
<span class="sourceLineNo">539</span>        // Log<a name="line.539"></a>
<span class="sourceLineNo">540</span>        this.info("starting " + this + " in directory " + this.logDir);<a name="line.540"></a>
<span class="sourceLineNo">541</span><a name="line.541"></a>
<span class="sourceLineNo">542</span>        // Start up local database<a name="line.542"></a>
<span class="sourceLineNo">543</span>        boolean success = false;<a name="line.543"></a>
<span class="sourceLineNo">544</span>        try {<a name="line.544"></a>
<span class="sourceLineNo">545</span><a name="line.545"></a>
<span class="sourceLineNo">546</span>            // Create/verify log directory<a name="line.546"></a>
<span class="sourceLineNo">547</span>            if (!this.logDir.exists() &amp;&amp; !this.logDir.isDirectory())<a name="line.547"></a>
<span class="sourceLineNo">548</span>                throw new IOException("failed to create directory `" + this.logDir + "'");<a name="line.548"></a>
<span class="sourceLineNo">549</span>            if (!this.logDir.isDirectory())<a name="line.549"></a>
<span class="sourceLineNo">550</span>                throw new IOException("file `" + this.logDir + "' is not a directory");<a name="line.550"></a>
<span class="sourceLineNo">551</span><a name="line.551"></a>
<span class="sourceLineNo">552</span>            // By default use an atomic key/value store based on LevelDB if not explicitly specified<a name="line.552"></a>
<span class="sourceLineNo">553</span>            if (this.kvstore != null)<a name="line.553"></a>
<span class="sourceLineNo">554</span>                this.kv = this.kvstore;<a name="line.554"></a>
<span class="sourceLineNo">555</span>            else {<a name="line.555"></a>
<span class="sourceLineNo">556</span>                final File leveldbDir = new File(this.logDir, "levedb" + KVSTORE_FILE_SUFFIX);<a name="line.556"></a>
<span class="sourceLineNo">557</span>                if (!leveldbDir.exists() &amp;&amp; !leveldbDir.mkdirs())<a name="line.557"></a>
<span class="sourceLineNo">558</span>                    throw new IOException("failed to create directory `" + leveldbDir + "'");<a name="line.558"></a>
<span class="sourceLineNo">559</span>                if (!leveldbDir.isDirectory())<a name="line.559"></a>
<span class="sourceLineNo">560</span>                    throw new IOException("file `" + leveldbDir + "' is not a directory");<a name="line.560"></a>
<span class="sourceLineNo">561</span>                this.kv = new LevelDBKVStore(new Iq80DBFactory().open(leveldbDir, new Options().createIfMissing(true)));<a name="line.561"></a>
<span class="sourceLineNo">562</span>            }<a name="line.562"></a>
<span class="sourceLineNo">563</span><a name="line.563"></a>
<span class="sourceLineNo">564</span>            // Open directory containing log entry files so we have a way to fsync() it<a name="line.564"></a>
<span class="sourceLineNo">565</span>            assert this.logDirChannel == null;<a name="line.565"></a>
<span class="sourceLineNo">566</span>            this.logDirChannel = FileChannel.open(this.logDir.toPath(), StandardOpenOption.READ);<a name="line.566"></a>
<span class="sourceLineNo">567</span><a name="line.567"></a>
<span class="sourceLineNo">568</span>            // Create randomizer<a name="line.568"></a>
<span class="sourceLineNo">569</span>            assert this.random == null;<a name="line.569"></a>
<span class="sourceLineNo">570</span>            this.random = new SecureRandom();<a name="line.570"></a>
<span class="sourceLineNo">571</span><a name="line.571"></a>
<span class="sourceLineNo">572</span>            // Start up executor thread<a name="line.572"></a>
<span class="sourceLineNo">573</span>            assert this.executor == null;<a name="line.573"></a>
<span class="sourceLineNo">574</span>            this.executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {<a name="line.574"></a>
<span class="sourceLineNo">575</span>                @Override<a name="line.575"></a>
<span class="sourceLineNo">576</span>                public Thread newThread(Runnable action) {<a name="line.576"></a>
<span class="sourceLineNo">577</span>                    return new ServiceThread(action);<a name="line.577"></a>
<span class="sourceLineNo">578</span>                }<a name="line.578"></a>
<span class="sourceLineNo">579</span>            });<a name="line.579"></a>
<span class="sourceLineNo">580</span><a name="line.580"></a>
<span class="sourceLineNo">581</span>            // Start network<a name="line.581"></a>
<span class="sourceLineNo">582</span>            this.network.start(new Network.Handler() {<a name="line.582"></a>
<span class="sourceLineNo">583</span>                @Override<a name="line.583"></a>
<span class="sourceLineNo">584</span>                public void handle(String sender, ByteBuffer buf) {<a name="line.584"></a>
<span class="sourceLineNo">585</span>                    RaftKVDatabase.this.handle(sender, buf);<a name="line.585"></a>
<span class="sourceLineNo">586</span>                }<a name="line.586"></a>
<span class="sourceLineNo">587</span><a name="line.587"></a>
<span class="sourceLineNo">588</span>                @Override<a name="line.588"></a>
<span class="sourceLineNo">589</span>                public void outputQueueEmpty(String address) {<a name="line.589"></a>
<span class="sourceLineNo">590</span>                    RaftKVDatabase.this.outputQueueEmpty(address);<a name="line.590"></a>
<span class="sourceLineNo">591</span>                }<a name="line.591"></a>
<span class="sourceLineNo">592</span>            });<a name="line.592"></a>
<span class="sourceLineNo">593</span><a name="line.593"></a>
<span class="sourceLineNo">594</span>            // Reload persistent raft info<a name="line.594"></a>
<span class="sourceLineNo">595</span>            this.clusterId = (int)this.decodeLong(CLUSTER_ID_KEY, 0);<a name="line.595"></a>
<span class="sourceLineNo">596</span>            this.currentTerm = this.decodeLong(CURRENT_TERM_KEY, 0);<a name="line.596"></a>
<span class="sourceLineNo">597</span>            final String votedFor = this.decodeString(VOTED_FOR_KEY, null);<a name="line.597"></a>
<span class="sourceLineNo">598</span>            this.lastAppliedTerm = this.decodeLong(LAST_APPLIED_TERM_KEY, 0);<a name="line.598"></a>
<span class="sourceLineNo">599</span>            this.lastAppliedIndex = this.decodeLong(LAST_APPLIED_INDEX_KEY, 0);<a name="line.599"></a>
<span class="sourceLineNo">600</span>            this.lastAppliedConfig = this.decodeConfig(LAST_APPLIED_CONFIG_KEY);<a name="line.600"></a>
<span class="sourceLineNo">601</span>            this.currentConfig = this.buildCurrentConfig();<a name="line.601"></a>
<span class="sourceLineNo">602</span><a name="line.602"></a>
<span class="sourceLineNo">603</span>            // If we crashed part way through a snapshot install, recover by resetting state machine<a name="line.603"></a>
<span class="sourceLineNo">604</span>            if (this.lastAppliedTerm &lt; 0 || this.lastAppliedIndex &lt; 0) {<a name="line.604"></a>
<span class="sourceLineNo">605</span>                this.info("detected partially applied snapshot, resetting state machine");<a name="line.605"></a>
<span class="sourceLineNo">606</span>                if (!this.resetStateMachine(true))<a name="line.606"></a>
<span class="sourceLineNo">607</span>                    throw new IOException("error resetting state machine");<a name="line.607"></a>
<span class="sourceLineNo">608</span>            }<a name="line.608"></a>
<span class="sourceLineNo">609</span><a name="line.609"></a>
<span class="sourceLineNo">610</span>            // Initialize commit index<a name="line.610"></a>
<span class="sourceLineNo">611</span>            this.commitIndex = this.lastAppliedIndex;<a name="line.611"></a>
<span class="sourceLineNo">612</span><a name="line.612"></a>
<span class="sourceLineNo">613</span>            // Reload outstanding log entries from disk<a name="line.613"></a>
<span class="sourceLineNo">614</span>            this.loadLog();<a name="line.614"></a>
<span class="sourceLineNo">615</span><a name="line.615"></a>
<span class="sourceLineNo">616</span>            // Show recovered state<a name="line.616"></a>
<span class="sourceLineNo">617</span>            this.info("recovered Raft state:"<a name="line.617"></a>
<span class="sourceLineNo">618</span>              + "\n  clusterId=" + (this.clusterId != 0 ? String.format("0x%08x", this.clusterId) : "none")<a name="line.618"></a>
<span class="sourceLineNo">619</span>              + "\n  currentTerm=" + this.currentTerm<a name="line.619"></a>
<span class="sourceLineNo">620</span>              + "\n  lastApplied=" + this.lastAppliedTerm + "/" + this.lastAppliedIndex<a name="line.620"></a>
<span class="sourceLineNo">621</span>              + "\n  lastAppliedConfig=" + this.lastAppliedConfig<a name="line.621"></a>
<span class="sourceLineNo">622</span>              + "\n  currentConfig=" + this.currentConfig<a name="line.622"></a>
<span class="sourceLineNo">623</span>              + "\n  votedFor=" + (votedFor != null ? "\"" + votedFor + "\"" : "nobody")<a name="line.623"></a>
<span class="sourceLineNo">624</span>              + "\n  log=" + this.raftLog);<a name="line.624"></a>
<span class="sourceLineNo">625</span><a name="line.625"></a>
<span class="sourceLineNo">626</span>            // Validate recovered state<a name="line.626"></a>
<span class="sourceLineNo">627</span>            if (this.isConfigured()) {<a name="line.627"></a>
<span class="sourceLineNo">628</span>                Preconditions.checkArgument(this.clusterId != 0);<a name="line.628"></a>
<span class="sourceLineNo">629</span>                Preconditions.checkArgument(this.currentTerm &gt; 0);<a name="line.629"></a>
<span class="sourceLineNo">630</span>                Preconditions.checkArgument(this.getLastLogTerm() &gt; 0);<a name="line.630"></a>
<span class="sourceLineNo">631</span>                Preconditions.checkArgument(this.getLastLogIndex() &gt; 0);<a name="line.631"></a>
<span class="sourceLineNo">632</span>                Preconditions.checkArgument(!this.currentConfig.isEmpty());<a name="line.632"></a>
<span class="sourceLineNo">633</span>            } else {<a name="line.633"></a>
<span class="sourceLineNo">634</span>                Preconditions.checkArgument(this.lastAppliedTerm == 0);<a name="line.634"></a>
<span class="sourceLineNo">635</span>                Preconditions.checkArgument(this.lastAppliedIndex == 0);<a name="line.635"></a>
<span class="sourceLineNo">636</span>                Preconditions.checkArgument(this.currentTerm == 0);<a name="line.636"></a>
<span class="sourceLineNo">637</span>                Preconditions.checkArgument(this.getLastLogTerm() == 0);<a name="line.637"></a>
<span class="sourceLineNo">638</span>                Preconditions.checkArgument(this.getLastLogIndex() == 0);<a name="line.638"></a>
<span class="sourceLineNo">639</span>                Preconditions.checkArgument(this.currentConfig.isEmpty());<a name="line.639"></a>
<span class="sourceLineNo">640</span>                Preconditions.checkArgument(this.raftLog.isEmpty());<a name="line.640"></a>
<span class="sourceLineNo">641</span>            }<a name="line.641"></a>
<span class="sourceLineNo">642</span><a name="line.642"></a>
<span class="sourceLineNo">643</span>            // Start as follower (with unknown leader)<a name="line.643"></a>
<span class="sourceLineNo">644</span>            this.changeRole(new FollowerRole(this, null, null, votedFor));<a name="line.644"></a>
<span class="sourceLineNo">645</span><a name="line.645"></a>
<span class="sourceLineNo">646</span>            // Done<a name="line.646"></a>
<span class="sourceLineNo">647</span>            success = true;<a name="line.647"></a>
<span class="sourceLineNo">648</span>        } finally {<a name="line.648"></a>
<span class="sourceLineNo">649</span>            if (!success)<a name="line.649"></a>
<span class="sourceLineNo">650</span>                this.cleanup();<a name="line.650"></a>
<span class="sourceLineNo">651</span>        }<a name="line.651"></a>
<span class="sourceLineNo">652</span><a name="line.652"></a>
<span class="sourceLineNo">653</span>        // Sanity check<a name="line.653"></a>
<span class="sourceLineNo">654</span>        assert this.checkState();<a name="line.654"></a>
<span class="sourceLineNo">655</span>    }<a name="line.655"></a>
<span class="sourceLineNo">656</span><a name="line.656"></a>
<span class="sourceLineNo">657</span>    /**<a name="line.657"></a>
<span class="sourceLineNo">658</span>     * Stop this instance.<a name="line.658"></a>
<span class="sourceLineNo">659</span>     *<a name="line.659"></a>
<span class="sourceLineNo">660</span>     * &lt;p&gt;<a name="line.660"></a>
<span class="sourceLineNo">661</span>     * Does nothing if not {@linkplain #start started} or in the process of shutting down.<a name="line.661"></a>
<span class="sourceLineNo">662</span>     * &lt;/p&gt;<a name="line.662"></a>
<span class="sourceLineNo">663</span>     */<a name="line.663"></a>
<span class="sourceLineNo">664</span>    @PreDestroy<a name="line.664"></a>
<span class="sourceLineNo">665</span>    public void stop() {<a name="line.665"></a>
<span class="sourceLineNo">666</span><a name="line.666"></a>
<span class="sourceLineNo">667</span>        // Set flag to prevent new transactions<a name="line.667"></a>
<span class="sourceLineNo">668</span>        synchronized (this) {<a name="line.668"></a>
<span class="sourceLineNo">669</span><a name="line.669"></a>
<span class="sourceLineNo">670</span>            // Sanity check<a name="line.670"></a>
<span class="sourceLineNo">671</span>            assert this.checkState();<a name="line.671"></a>
<span class="sourceLineNo">672</span>            if (this.role == null || this.shuttingDown)<a name="line.672"></a>
<span class="sourceLineNo">673</span>                return;<a name="line.673"></a>
<span class="sourceLineNo">674</span><a name="line.674"></a>
<span class="sourceLineNo">675</span>            // Set shutting down flag<a name="line.675"></a>
<span class="sourceLineNo">676</span>            this.info("starting shutdown of " + this);<a name="line.676"></a>
<span class="sourceLineNo">677</span>            this.shuttingDown = true;<a name="line.677"></a>
<span class="sourceLineNo">678</span><a name="line.678"></a>
<span class="sourceLineNo">679</span>            // Fail all remaining open transactions<a name="line.679"></a>
<span class="sourceLineNo">680</span>            for (RaftKVTransaction tx : new ArrayList&lt;RaftKVTransaction&gt;(this.openTransactions.values())) {<a name="line.680"></a>
<span class="sourceLineNo">681</span>                switch (tx.getState()) {<a name="line.681"></a>
<span class="sourceLineNo">682</span>                case EXECUTING:<a name="line.682"></a>
<span class="sourceLineNo">683</span>                    tx.rollback();<a name="line.683"></a>
<span class="sourceLineNo">684</span>                    break;<a name="line.684"></a>
<span class="sourceLineNo">685</span>                case COMMIT_READY:<a name="line.685"></a>
<span class="sourceLineNo">686</span>                case COMMIT_WAITING:<a name="line.686"></a>
<span class="sourceLineNo">687</span>                    this.fail(tx, new KVTransactionException(tx, "database shutdown"));<a name="line.687"></a>
<span class="sourceLineNo">688</span>                    break;<a name="line.688"></a>
<span class="sourceLineNo">689</span>                case COMPLETED:<a name="line.689"></a>
<span class="sourceLineNo">690</span>                    break;<a name="line.690"></a>
<span class="sourceLineNo">691</span>                default:<a name="line.691"></a>
<span class="sourceLineNo">692</span>                    assert false;<a name="line.692"></a>
<span class="sourceLineNo">693</span>                    break;<a name="line.693"></a>
<span class="sourceLineNo">694</span>                }<a name="line.694"></a>
<span class="sourceLineNo">695</span>            }<a name="line.695"></a>
<span class="sourceLineNo">696</span><a name="line.696"></a>
<span class="sourceLineNo">697</span>            // Sleep while we wait for transactions to clean themselves up<a name="line.697"></a>
<span class="sourceLineNo">698</span>            boolean done = false;<a name="line.698"></a>
<span class="sourceLineNo">699</span>            try {<a name="line.699"></a>
<span class="sourceLineNo">700</span>                done = TimedWait.wait(this, 5000, new org.dellroad.stuff.java.Predicate() {<a name="line.700"></a>
<span class="sourceLineNo">701</span>                    @Override<a name="line.701"></a>
<span class="sourceLineNo">702</span>                    public boolean test() {<a name="line.702"></a>
<span class="sourceLineNo">703</span>                        return RaftKVDatabase.this.openTransactions.isEmpty();<a name="line.703"></a>
<span class="sourceLineNo">704</span>                    }<a name="line.704"></a>
<span class="sourceLineNo">705</span>                });<a name="line.705"></a>
<span class="sourceLineNo">706</span>            } catch (InterruptedException e) {<a name="line.706"></a>
<span class="sourceLineNo">707</span>                Thread.currentThread().interrupt();<a name="line.707"></a>
<span class="sourceLineNo">708</span>            }<a name="line.708"></a>
<span class="sourceLineNo">709</span>            if (!done)<a name="line.709"></a>
<span class="sourceLineNo">710</span>                this.warn("open transactions not cleaned up during shutdown");<a name="line.710"></a>
<span class="sourceLineNo">711</span>        }<a name="line.711"></a>
<span class="sourceLineNo">712</span><a name="line.712"></a>
<span class="sourceLineNo">713</span>        // Shut down the executor and wait for pending tasks to finish<a name="line.713"></a>
<span class="sourceLineNo">714</span>        this.executor.shutdownNow();<a name="line.714"></a>
<span class="sourceLineNo">715</span>        try {<a name="line.715"></a>
<span class="sourceLineNo">716</span>            this.executor.awaitTermination(1000, TimeUnit.MILLISECONDS);<a name="line.716"></a>
<span class="sourceLineNo">717</span>        } catch (InterruptedException e) {<a name="line.717"></a>
<span class="sourceLineNo">718</span>            Thread.currentThread().interrupt();<a name="line.718"></a>
<span class="sourceLineNo">719</span>        }<a name="line.719"></a>
<span class="sourceLineNo">720</span><a name="line.720"></a>
<span class="sourceLineNo">721</span>        // Final cleanup<a name="line.721"></a>
<span class="sourceLineNo">722</span>        synchronized (this) {<a name="line.722"></a>
<span class="sourceLineNo">723</span>            this.executor = null;<a name="line.723"></a>
<span class="sourceLineNo">724</span>            this.cleanup();<a name="line.724"></a>
<span class="sourceLineNo">725</span>        }<a name="line.725"></a>
<span class="sourceLineNo">726</span><a name="line.726"></a>
<span class="sourceLineNo">727</span>        // Done<a name="line.727"></a>
<span class="sourceLineNo">728</span>        this.info("completed shutdown of " + this);<a name="line.728"></a>
<span class="sourceLineNo">729</span>    }<a name="line.729"></a>
<span class="sourceLineNo">730</span><a name="line.730"></a>
<span class="sourceLineNo">731</span>    private void cleanup() {<a name="line.731"></a>
<span class="sourceLineNo">732</span>        assert Thread.holdsLock(this);<a name="line.732"></a>
<span class="sourceLineNo">733</span>        assert this.openTransactions.isEmpty();<a name="line.733"></a>
<span class="sourceLineNo">734</span>        if (this.role != null) {<a name="line.734"></a>
<span class="sourceLineNo">735</span>            this.role.shutdown();<a name="line.735"></a>
<span class="sourceLineNo">736</span>            this.role = null;<a name="line.736"></a>
<span class="sourceLineNo">737</span>        }<a name="line.737"></a>
<span class="sourceLineNo">738</span>        if (this.executor != null) {<a name="line.738"></a>
<span class="sourceLineNo">739</span>            this.executor.shutdownNow();<a name="line.739"></a>
<span class="sourceLineNo">740</span>            try {<a name="line.740"></a>
<span class="sourceLineNo">741</span>                this.executor.awaitTermination(1000, TimeUnit.MILLISECONDS);<a name="line.741"></a>
<span class="sourceLineNo">742</span>            } catch (InterruptedException e) {<a name="line.742"></a>
<span class="sourceLineNo">743</span>                Thread.currentThread().interrupt();<a name="line.743"></a>
<span class="sourceLineNo">744</span>            }<a name="line.744"></a>
<span class="sourceLineNo">745</span>            this.executor = null;<a name="line.745"></a>
<span class="sourceLineNo">746</span>        }<a name="line.746"></a>
<span class="sourceLineNo">747</span>        if (this.kv != null) {<a name="line.747"></a>
<span class="sourceLineNo">748</span>            if (this.kvstore == null) {<a name="line.748"></a>
<span class="sourceLineNo">749</span>                final LevelDBKVStore ldb = (LevelDBKVStore)this.kv;<a name="line.749"></a>
<span class="sourceLineNo">750</span>                Util.closeIfPossible(ldb);<a name="line.750"></a>
<span class="sourceLineNo">751</span>                Util.closeIfPossible(ldb.getDB());<a name="line.751"></a>
<span class="sourceLineNo">752</span>            }<a name="line.752"></a>
<span class="sourceLineNo">753</span>            this.kv = null;<a name="line.753"></a>
<span class="sourceLineNo">754</span>        }<a name="line.754"></a>
<span class="sourceLineNo">755</span>        Util.closeIfPossible(this.logDirChannel);<a name="line.755"></a>
<span class="sourceLineNo">756</span>        this.logDirChannel = null;<a name="line.756"></a>
<span class="sourceLineNo">757</span>        this.raftLog.clear();<a name="line.757"></a>
<span class="sourceLineNo">758</span>        this.random = null;<a name="line.758"></a>
<span class="sourceLineNo">759</span>        this.network.stop();<a name="line.759"></a>
<span class="sourceLineNo">760</span>        this.currentTerm = 0;<a name="line.760"></a>
<span class="sourceLineNo">761</span>        this.commitIndex = 0;<a name="line.761"></a>
<span class="sourceLineNo">762</span>        this.clusterId = 0;<a name="line.762"></a>
<span class="sourceLineNo">763</span>        this.lastAppliedTerm = 0;<a name="line.763"></a>
<span class="sourceLineNo">764</span>        this.lastAppliedIndex = 0;<a name="line.764"></a>
<span class="sourceLineNo">765</span>        this.lastAppliedConfig = null;<a name="line.765"></a>
<span class="sourceLineNo">766</span>        this.currentConfig = null;<a name="line.766"></a>
<span class="sourceLineNo">767</span>        this.transmitting.clear();<a name="line.767"></a>
<span class="sourceLineNo">768</span>        this.pendingService.clear();<a name="line.768"></a>
<span class="sourceLineNo">769</span>        this.shuttingDown = false;<a name="line.769"></a>
<span class="sourceLineNo">770</span>    }<a name="line.770"></a>
<span class="sourceLineNo">771</span><a name="line.771"></a>
<span class="sourceLineNo">772</span>    /**<a name="line.772"></a>
<span class="sourceLineNo">773</span>     * Initialize our in-memory state from the persistent state reloaded from disk.<a name="line.773"></a>
<span class="sourceLineNo">774</span>     * This is invoked on initial startup.<a name="line.774"></a>
<span class="sourceLineNo">775</span>     */<a name="line.775"></a>
<span class="sourceLineNo">776</span>    private void loadLog() throws IOException {<a name="line.776"></a>
<span class="sourceLineNo">777</span><a name="line.777"></a>
<span class="sourceLineNo">778</span>        // Sanity check<a name="line.778"></a>
<span class="sourceLineNo">779</span>        assert Thread.holdsLock(this);<a name="line.779"></a>
<span class="sourceLineNo">780</span><a name="line.780"></a>
<span class="sourceLineNo">781</span>        // Scan for log entry files<a name="line.781"></a>
<span class="sourceLineNo">782</span>        this.raftLog.clear();<a name="line.782"></a>
<span class="sourceLineNo">783</span>        for (File file : this.logDir.listFiles()) {<a name="line.783"></a>
<span class="sourceLineNo">784</span><a name="line.784"></a>
<span class="sourceLineNo">785</span>            // Is this a log entry file?<a name="line.785"></a>
<span class="sourceLineNo">786</span>            if (LogEntry.LOG_FILE_PATTERN.matcher(file.getName()).matches()) {<a name="line.786"></a>
<span class="sourceLineNo">787</span>                this.info("recovering log file " + file.getName());<a name="line.787"></a>
<span class="sourceLineNo">788</span>                this.raftLog.add(LogEntry.fromFile(file));<a name="line.788"></a>
<span class="sourceLineNo">789</span>                continue;<a name="line.789"></a>
<span class="sourceLineNo">790</span>            }<a name="line.790"></a>
<span class="sourceLineNo">791</span><a name="line.791"></a>
<span class="sourceLineNo">792</span>            // Is this a leftover temporary file?<a name="line.792"></a>
<span class="sourceLineNo">793</span>            if (TEMP_FILE_PATTERN.matcher(file.getName()).matches()) {<a name="line.793"></a>
<span class="sourceLineNo">794</span>                this.info("deleting leftover temporary file " + file.getName());<a name="line.794"></a>
<span class="sourceLineNo">795</span>                if (!file.delete())<a name="line.795"></a>
<span class="sourceLineNo">796</span>                    this.error("failed to delete leftover temporary file " + file);<a name="line.796"></a>
<span class="sourceLineNo">797</span>                continue;<a name="line.797"></a>
<span class="sourceLineNo">798</span>            }<a name="line.798"></a>
<span class="sourceLineNo">799</span><a name="line.799"></a>
<span class="sourceLineNo">800</span>            // Is this a KV store directory (expected)?<a name="line.800"></a>
<span class="sourceLineNo">801</span>            if (KVSTORE_FILE_PATTERN.matcher(file.getName()).matches())<a name="line.801"></a>
<span class="sourceLineNo">802</span>                continue;<a name="line.802"></a>
<span class="sourceLineNo">803</span><a name="line.803"></a>
<span class="sourceLineNo">804</span>            // Unknown<a name="line.804"></a>
<span class="sourceLineNo">805</span>            this.warn("ignoring unrecognized file " + file.getName() + " in my log directory");<a name="line.805"></a>
<span class="sourceLineNo">806</span>        }<a name="line.806"></a>
<span class="sourceLineNo">807</span><a name="line.807"></a>
<span class="sourceLineNo">808</span>        // Verify we have a contiguous range of log entries starting from the snapshot index; discard bogus log files<a name="line.808"></a>
<span class="sourceLineNo">809</span>        Collections.sort(this.raftLog, LogEntry.SORT_BY_INDEX);<a name="line.809"></a>
<span class="sourceLineNo">810</span>        long lastTermSeen = this.lastAppliedTerm;<a name="line.810"></a>
<span class="sourceLineNo">811</span>        long expectedIndex = this.lastAppliedIndex + 1;<a name="line.811"></a>
<span class="sourceLineNo">812</span>        for (Iterator&lt;LogEntry&gt; i = this.raftLog.iterator(); i.hasNext(); ) {<a name="line.812"></a>
<span class="sourceLineNo">813</span>            final LogEntry logEntry = i.next();<a name="line.813"></a>
<span class="sourceLineNo">814</span>            String error = null;<a name="line.814"></a>
<span class="sourceLineNo">815</span>            if (logEntry.getTerm() &lt; lastTermSeen)<a name="line.815"></a>
<span class="sourceLineNo">816</span>                error = "term " + logEntry.getTerm() + " &lt; last applied term " + lastTermSeen;<a name="line.816"></a>
<span class="sourceLineNo">817</span>            else if (logEntry.getIndex() &lt; this.lastAppliedIndex)<a name="line.817"></a>
<span class="sourceLineNo">818</span>                error = "index " + logEntry.getIndex() + " &lt; last applied index " + this.lastAppliedIndex;<a name="line.818"></a>
<span class="sourceLineNo">819</span>            else if (logEntry.getIndex() != expectedIndex)<a name="line.819"></a>
<span class="sourceLineNo">820</span>                error = "index " + logEntry.getIndex() + " != expected index " + expectedIndex;<a name="line.820"></a>
<span class="sourceLineNo">821</span>            if (error != null) {<a name="line.821"></a>
<span class="sourceLineNo">822</span>                this.warn("deleting bogus log file " + logEntry.getFile().getName() + ": " + error);<a name="line.822"></a>
<span class="sourceLineNo">823</span>                if (!logEntry.getFile().delete())<a name="line.823"></a>
<span class="sourceLineNo">824</span>                    this.error("failed to delete bogus log file " + logEntry.getFile());<a name="line.824"></a>
<span class="sourceLineNo">825</span>                i.remove();<a name="line.825"></a>
<span class="sourceLineNo">826</span>            } else {<a name="line.826"></a>
<span class="sourceLineNo">827</span>                expectedIndex++;<a name="line.827"></a>
<span class="sourceLineNo">828</span>                lastTermSeen = logEntry.getTerm();<a name="line.828"></a>
<span class="sourceLineNo">829</span>            }<a name="line.829"></a>
<span class="sourceLineNo">830</span>        }<a name="line.830"></a>
<span class="sourceLineNo">831</span>        this.info("recovered " + this.raftLog.size() + " Raft log entries: " + this.raftLog);<a name="line.831"></a>
<span class="sourceLineNo">832</span><a name="line.832"></a>
<span class="sourceLineNo">833</span>        // Rebuild current configuration<a name="line.833"></a>
<span class="sourceLineNo">834</span>        this.currentConfig = this.buildCurrentConfig();<a name="line.834"></a>
<span class="sourceLineNo">835</span>    }<a name="line.835"></a>
<span class="sourceLineNo">836</span><a name="line.836"></a>
<span class="sourceLineNo">837</span>    private Map&lt;String, String&gt; buildCurrentConfig() {<a name="line.837"></a>
<span class="sourceLineNo">838</span><a name="line.838"></a>
<span class="sourceLineNo">839</span>        // Start with last applied config<a name="line.839"></a>
<span class="sourceLineNo">840</span>        final HashMap&lt;String, String&gt; config = new HashMap&lt;&gt;(this.lastAppliedConfig);<a name="line.840"></a>
<span class="sourceLineNo">841</span><a name="line.841"></a>
<span class="sourceLineNo">842</span>        // Apply any changes found in uncommitted log entries<a name="line.842"></a>
<span class="sourceLineNo">843</span>        for (LogEntry logEntry : this.raftLog)<a name="line.843"></a>
<span class="sourceLineNo">844</span>            logEntry.applyConfigChange(config);<a name="line.844"></a>
<span class="sourceLineNo">845</span><a name="line.845"></a>
<span class="sourceLineNo">846</span>        // Done<a name="line.846"></a>
<span class="sourceLineNo">847</span>        return config;<a name="line.847"></a>
<span class="sourceLineNo">848</span>    }<a name="line.848"></a>
<span class="sourceLineNo">849</span><a name="line.849"></a>
<span class="sourceLineNo">850</span>// Configuration Stuff<a name="line.850"></a>
<span class="sourceLineNo">851</span><a name="line.851"></a>
<span class="sourceLineNo">852</span>    /**<a name="line.852"></a>
<span class="sourceLineNo">853</span>     * Retrieve the unique 32-bit ID for this node's cluster.<a name="line.853"></a>
<span class="sourceLineNo">854</span>     *<a name="line.854"></a>
<span class="sourceLineNo">855</span>     * &lt;p&gt;<a name="line.855"></a>
<span class="sourceLineNo">856</span>     * A value of zero indicates an unconfigured system. Usually the reverse true, though an unconfigured system<a name="line.856"></a>
<span class="sourceLineNo">857</span>     * can have a non-zero cluster ID in the rare case where an error occurred persisting the initial log entry.<a name="line.857"></a>
<span class="sourceLineNo">858</span>     *<a name="line.858"></a>
<span class="sourceLineNo">859</span>     * @return unique cluster ID, or zero if this node is unconfigured<a name="line.859"></a>
<span class="sourceLineNo">860</span>     */<a name="line.860"></a>
<span class="sourceLineNo">861</span>    public synchronized int getClusterId() {<a name="line.861"></a>
<span class="sourceLineNo">862</span>        return this.clusterId;<a name="line.862"></a>
<span class="sourceLineNo">863</span>    }<a name="line.863"></a>
<span class="sourceLineNo">864</span><a name="line.864"></a>
<span class="sourceLineNo">865</span>    /**<a name="line.865"></a>
<span class="sourceLineNo">866</span>     * Retrieve the current cluster configuration as understood by this node.<a name="line.866"></a>
<span class="sourceLineNo">867</span>     *<a name="line.867"></a>
<span class="sourceLineNo">868</span>     * &lt;p&gt;<a name="line.868"></a>
<span class="sourceLineNo">869</span>     * Configuration changes are performed and committed in the context of a normal transaction; see<a name="line.869"></a>
<span class="sourceLineNo">870</span>     * {@link RaftKVTransaction#configChange RaftKVTransaction.configChange()}.<a name="line.870"></a>
<span class="sourceLineNo">871</span>     *<a name="line.871"></a>
<span class="sourceLineNo">872</span>     * &lt;p&gt;<a name="line.872"></a>
<span class="sourceLineNo">873</span>     * If this system is unconfigured, an emtyp map is returned (and vice-versa).<a name="line.873"></a>
<span class="sourceLineNo">874</span>     *<a name="line.874"></a>
<span class="sourceLineNo">875</span>     * @return current configuration mapping from node identity to network address, or empty if this node is unconfigured<a name="line.875"></a>
<span class="sourceLineNo">876</span>     */<a name="line.876"></a>
<span class="sourceLineNo">877</span>    public synchronized Map&lt;String, String&gt; getCurrentConfig() {<a name="line.877"></a>
<span class="sourceLineNo">878</span>        return new HashMap&lt;&gt;(this.currentConfig);<a name="line.878"></a>
<span class="sourceLineNo">879</span>    }<a name="line.879"></a>
<span class="sourceLineNo">880</span><a name="line.880"></a>
<span class="sourceLineNo">881</span>    /**<a name="line.881"></a>
<span class="sourceLineNo">882</span>     * Determine whether this instance is configured.<a name="line.882"></a>
<span class="sourceLineNo">883</span>     *<a name="line.883"></a>
<span class="sourceLineNo">884</span>     * &lt;p&gt;<a name="line.884"></a>
<span class="sourceLineNo">885</span>     * A node is configured iff it has added at least one Raft log entry (because the first log entry in any<a name="line.885"></a>
<span class="sourceLineNo">886</span>     * new cluster always includes a configuration change that adds the first node in the cluster).<a name="line.886"></a>
<span class="sourceLineNo">887</span>     *<a name="line.887"></a>
<span class="sourceLineNo">888</span>     * @return true if this instance is configured, otherwise false<a name="line.888"></a>
<span class="sourceLineNo">889</span>     */<a name="line.889"></a>
<span class="sourceLineNo">890</span>    public boolean isConfigured() {<a name="line.890"></a>
<span class="sourceLineNo">891</span>        return this.lastAppliedIndex &gt; 0 || !this.raftLog.isEmpty();<a name="line.891"></a>
<span class="sourceLineNo">892</span>    }<a name="line.892"></a>
<span class="sourceLineNo">893</span><a name="line.893"></a>
<span class="sourceLineNo">894</span>    /**<a name="line.894"></a>
<span class="sourceLineNo">895</span>     * Determine whether this node thinks that it is part of the cluster, as currently configured on this node.<a name="line.895"></a>
<span class="sourceLineNo">896</span>     *<a name="line.896"></a>
<span class="sourceLineNo">897</span>     * @return true if this instance is part of the cluster, otherwise false<a name="line.897"></a>
<span class="sourceLineNo">898</span>     */<a name="line.898"></a>
<span class="sourceLineNo">899</span>    public boolean isClusterMember() {<a name="line.899"></a>
<span class="sourceLineNo">900</span>        return this.isClusterMember(this.identity);<a name="line.900"></a>
<span class="sourceLineNo">901</span>    }<a name="line.901"></a>
<span class="sourceLineNo">902</span><a name="line.902"></a>
<span class="sourceLineNo">903</span>    /**<a name="line.903"></a>
<span class="sourceLineNo">904</span>     * Determine whether this node thinks that the specified node is part of the cluster, as currently configured on this node.<a name="line.904"></a>
<span class="sourceLineNo">905</span>     *<a name="line.905"></a>
<span class="sourceLineNo">906</span>     * @param node node identity<a name="line.906"></a>
<span class="sourceLineNo">907</span>     * @return true if the specified node is part of the cluster, otherwise false<a name="line.907"></a>
<span class="sourceLineNo">908</span>     */<a name="line.908"></a>
<span class="sourceLineNo">909</span>    public boolean isClusterMember(String node) {<a name="line.909"></a>
<span class="sourceLineNo">910</span>        return this.currentConfig.containsKey(node);<a name="line.910"></a>
<span class="sourceLineNo">911</span>    }<a name="line.911"></a>
<span class="sourceLineNo">912</span><a name="line.912"></a>
<span class="sourceLineNo">913</span>// Transactions<a name="line.913"></a>
<span class="sourceLineNo">914</span><a name="line.914"></a>
<span class="sourceLineNo">915</span>    /**<a name="line.915"></a>
<span class="sourceLineNo">916</span>     * Create a new transaction.<a name="line.916"></a>
<span class="sourceLineNo">917</span>     *<a name="line.917"></a>
<span class="sourceLineNo">918</span>     * @throws IllegalStateException if this instance is not {@linkplain #start started} or in the process of shutting down<a name="line.918"></a>
<span class="sourceLineNo">919</span>     */<a name="line.919"></a>
<span class="sourceLineNo">920</span>    @Override<a name="line.920"></a>
<span class="sourceLineNo">921</span>    public synchronized RaftKVTransaction createTransaction() {<a name="line.921"></a>
<span class="sourceLineNo">922</span><a name="line.922"></a>
<span class="sourceLineNo">923</span>        // Sanity check<a name="line.923"></a>
<span class="sourceLineNo">924</span>        assert this.checkState();<a name="line.924"></a>
<span class="sourceLineNo">925</span>        Preconditions.checkState(this.role != null, "not started");<a name="line.925"></a>
<span class="sourceLineNo">926</span>        Preconditions.checkState(!this.shuttingDown, "shutting down");<a name="line.926"></a>
<span class="sourceLineNo">927</span><a name="line.927"></a>
<span class="sourceLineNo">928</span>        // Base transaction on the most recent log entry. This is itself a form of optimistic locking: we assume that the<a name="line.928"></a>
<span class="sourceLineNo">929</span>        // most recent log entry has a high probability of being committed (in the Raft sense), which is of course required<a name="line.929"></a>
<span class="sourceLineNo">930</span>        // in order to commit any transaction based on it.<a name="line.930"></a>
<span class="sourceLineNo">931</span>        final MostRecentView view = new MostRecentView();<a name="line.931"></a>
<span class="sourceLineNo">932</span><a name="line.932"></a>
<span class="sourceLineNo">933</span>        // Create transaction<a name="line.933"></a>
<span class="sourceLineNo">934</span>        final RaftKVTransaction tx = new RaftKVTransaction(this,<a name="line.934"></a>
<span class="sourceLineNo">935</span>          this.getLastLogTerm(), this.getLastLogIndex(), view.getSnapshot(), view.getView());<a name="line.935"></a>
<span class="sourceLineNo">936</span>        if (this.log.isDebugEnabled())<a name="line.936"></a>
<span class="sourceLineNo">937</span>            this.debug("created new transaction " + tx);<a name="line.937"></a>
<span class="sourceLineNo">938</span>        this.openTransactions.put(tx.getTxId(), tx);<a name="line.938"></a>
<span class="sourceLineNo">939</span><a name="line.939"></a>
<span class="sourceLineNo">940</span>        // Set default commit timeout<a name="line.940"></a>
<span class="sourceLineNo">941</span>        tx.setTimeout(this.commitTimeout);<a name="line.941"></a>
<span class="sourceLineNo">942</span><a name="line.942"></a>
<span class="sourceLineNo">943</span>        // Done<a name="line.943"></a>
<span class="sourceLineNo">944</span>        return tx;<a name="line.944"></a>
<span class="sourceLineNo">945</span>    }<a name="line.945"></a>
<span class="sourceLineNo">946</span><a name="line.946"></a>
<span class="sourceLineNo">947</span>    /**<a name="line.947"></a>
<span class="sourceLineNo">948</span>     * Commit a transaction.<a name="line.948"></a>
<span class="sourceLineNo">949</span>     */<a name="line.949"></a>
<span class="sourceLineNo">950</span>    void commit(final RaftKVTransaction tx) {<a name="line.950"></a>
<span class="sourceLineNo">951</span>        try {<a name="line.951"></a>
<span class="sourceLineNo">952</span><a name="line.952"></a>
<span class="sourceLineNo">953</span>            // Mark transaction as "commit ready" - service thread will do the rest<a name="line.953"></a>
<span class="sourceLineNo">954</span>            synchronized (this) {<a name="line.954"></a>
<span class="sourceLineNo">955</span><a name="line.955"></a>
<span class="sourceLineNo">956</span>                // Sanity check<a name="line.956"></a>
<span class="sourceLineNo">957</span>                assert this.checkState();<a name="line.957"></a>
<span class="sourceLineNo">958</span>                assert this.role != null;<a name="line.958"></a>
<span class="sourceLineNo">959</span><a name="line.959"></a>
<span class="sourceLineNo">960</span>                // Check tx state<a name="line.960"></a>
<span class="sourceLineNo">961</span>                switch (tx.getState()) {<a name="line.961"></a>
<span class="sourceLineNo">962</span>                case EXECUTING:<a name="line.962"></a>
<span class="sourceLineNo">963</span><a name="line.963"></a>
<span class="sourceLineNo">964</span>                    // Transition to COMMIT_READY state<a name="line.964"></a>
<span class="sourceLineNo">965</span>                    if (this.log.isDebugEnabled())<a name="line.965"></a>
<span class="sourceLineNo">966</span>                        this.debug("committing transaction " + tx);<a name="line.966"></a>
<span class="sourceLineNo">967</span>                    tx.closeSnapshot();<a name="line.967"></a>
<span class="sourceLineNo">968</span>                    tx.setState(TxState.COMMIT_READY);<a name="line.968"></a>
<span class="sourceLineNo">969</span>                    this.requestService(this.role.new CheckReadyTransactionService(tx));<a name="line.969"></a>
<span class="sourceLineNo">970</span><a name="line.970"></a>
<span class="sourceLineNo">971</span>                    // Setup commit timer<a name="line.971"></a>
<span class="sourceLineNo">972</span>                    final int timeout = tx.getTimeout();<a name="line.972"></a>
<span class="sourceLineNo">973</span>                    if (timeout != 0) {<a name="line.973"></a>
<span class="sourceLineNo">974</span>                        final Timer commitTimer = new Timer("commit timer for " + tx,<a name="line.974"></a>
<span class="sourceLineNo">975</span>                          new Service(null, "commit timeout for tx#" + tx.getTxId()) {<a name="line.975"></a>
<span class="sourceLineNo">976</span>                            @Override<a name="line.976"></a>
<span class="sourceLineNo">977</span>                            public void run() {<a name="line.977"></a>
<span class="sourceLineNo">978</span>                                switch (tx.getState()) {<a name="line.978"></a>
<span class="sourceLineNo">979</span>                                case COMMIT_READY:<a name="line.979"></a>
<span class="sourceLineNo">980</span>                                case COMMIT_WAITING:<a name="line.980"></a>
<span class="sourceLineNo">981</span>                                    RaftKVDatabase.this.fail(tx, new RetryTransactionException(tx,<a name="line.981"></a>
<span class="sourceLineNo">982</span>                                      "transaction failed to complete within " + timeout + "ms (in state " + tx.getState() + ")"));<a name="line.982"></a>
<span class="sourceLineNo">983</span>                                    break;<a name="line.983"></a>
<span class="sourceLineNo">984</span>                                default:<a name="line.984"></a>
<span class="sourceLineNo">985</span>                                    break;<a name="line.985"></a>
<span class="sourceLineNo">986</span>                                }<a name="line.986"></a>
<span class="sourceLineNo">987</span>                            }<a name="line.987"></a>
<span class="sourceLineNo">988</span>                        });<a name="line.988"></a>
<span class="sourceLineNo">989</span>                        commitTimer.timeoutAfter(tx.getTimeout());<a name="line.989"></a>
<span class="sourceLineNo">990</span>                        tx.setCommitTimer(commitTimer);<a name="line.990"></a>
<span class="sourceLineNo">991</span>                    }<a name="line.991"></a>
<span class="sourceLineNo">992</span>                    break;<a name="line.992"></a>
<span class="sourceLineNo">993</span>                case CLOSED:                                        // this transaction has already been committed or rolled back<a name="line.993"></a>
<span class="sourceLineNo">994</span>                    throw new StaleTransactionException(tx);<a name="line.994"></a>
<span class="sourceLineNo">995</span>                default:                                            // another thread is already doing the commit<a name="line.995"></a>
<span class="sourceLineNo">996</span>                    this.warn("simultaneous commit()'s requested for " + tx + " by two different threads");<a name="line.996"></a>
<span class="sourceLineNo">997</span>                    break;<a name="line.997"></a>
<span class="sourceLineNo">998</span>                }<a name="line.998"></a>
<span class="sourceLineNo">999</span>            }<a name="line.999"></a>
<span class="sourceLineNo">1000</span><a name="line.1000"></a>
<span class="sourceLineNo">1001</span>            // Wait for completion<a name="line.1001"></a>
<span class="sourceLineNo">1002</span>            try {<a name="line.1002"></a>
<span class="sourceLineNo">1003</span>                tx.getCommitFuture().get();<a name="line.1003"></a>
<span class="sourceLineNo">1004</span>            } catch (InterruptedException e) {<a name="line.1004"></a>
<span class="sourceLineNo">1005</span>                throw new RetryTransactionException(tx, "thread interrupted while waiting for commit", e);<a name="line.1005"></a>
<span class="sourceLineNo">1006</span>            } catch (ExecutionException e) {<a name="line.1006"></a>
<span class="sourceLineNo">1007</span>                final Throwable cause = e.getCause();<a name="line.1007"></a>
<span class="sourceLineNo">1008</span>                Util.prependCurrentStackTrace(cause, "Asynchronous Commit");<a name="line.1008"></a>
<span class="sourceLineNo">1009</span>                if (cause instanceof Error)<a name="line.1009"></a>
<span class="sourceLineNo">1010</span>                    throw (Error)cause;<a name="line.1010"></a>
<span class="sourceLineNo">1011</span>                if (cause instanceof RuntimeException)<a name="line.1011"></a>
<span class="sourceLineNo">1012</span>                    throw (RuntimeException)cause;<a name="line.1012"></a>
<span class="sourceLineNo">1013</span>                throw new KVTransactionException(tx, "commit failed", cause);           // should never get here<a name="line.1013"></a>
<span class="sourceLineNo">1014</span>            }<a name="line.1014"></a>
<span class="sourceLineNo">1015</span>        } finally {<a name="line.1015"></a>
<span class="sourceLineNo">1016</span>            this.cleanupTransaction(tx);<a name="line.1016"></a>
<span class="sourceLineNo">1017</span>        }<a name="line.1017"></a>
<span class="sourceLineNo">1018</span>    }<a name="line.1018"></a>
<span class="sourceLineNo">1019</span><a name="line.1019"></a>
<span class="sourceLineNo">1020</span>    /**<a name="line.1020"></a>
<span class="sourceLineNo">1021</span>     * Rollback a transaction.<a name="line.1021"></a>
<span class="sourceLineNo">1022</span>     */<a name="line.1022"></a>
<span class="sourceLineNo">1023</span>    synchronized void rollback(RaftKVTransaction tx) {<a name="line.1023"></a>
<span class="sourceLineNo">1024</span><a name="line.1024"></a>
<span class="sourceLineNo">1025</span>        // Sanity check<a name="line.1025"></a>
<span class="sourceLineNo">1026</span>        assert this.checkState();<a name="line.1026"></a>
<span class="sourceLineNo">1027</span>        assert this.role != null;<a name="line.1027"></a>
<span class="sourceLineNo">1028</span><a name="line.1028"></a>
<span class="sourceLineNo">1029</span>        // Check tx state<a name="line.1029"></a>
<span class="sourceLineNo">1030</span>        switch (tx.getState()) {<a name="line.1030"></a>
<span class="sourceLineNo">1031</span>        case EXECUTING:<a name="line.1031"></a>
<span class="sourceLineNo">1032</span>            if (this.log.isDebugEnabled())<a name="line.1032"></a>
<span class="sourceLineNo">1033</span>                this.debug("rolling back transaction " + tx);<a name="line.1033"></a>
<span class="sourceLineNo">1034</span>            this.cleanupTransaction(tx);<a name="line.1034"></a>
<span class="sourceLineNo">1035</span>            break;<a name="line.1035"></a>
<span class="sourceLineNo">1036</span>        case CLOSED:<a name="line.1036"></a>
<span class="sourceLineNo">1037</span>            break;<a name="line.1037"></a>
<span class="sourceLineNo">1038</span>        default:                                            // another thread is currently committing!<a name="line.1038"></a>
<span class="sourceLineNo">1039</span>            this.warn("simultaneous commit() and rollback() requested for " + tx + " by two different threads");<a name="line.1039"></a>
<span class="sourceLineNo">1040</span>            break;<a name="line.1040"></a>
<span class="sourceLineNo">1041</span>        }<a name="line.1041"></a>
<span class="sourceLineNo">1042</span>    }<a name="line.1042"></a>
<span class="sourceLineNo">1043</span><a name="line.1043"></a>
<span class="sourceLineNo">1044</span>    private synchronized void cleanupTransaction(RaftKVTransaction tx) {<a name="line.1044"></a>
<span class="sourceLineNo">1045</span><a name="line.1045"></a>
<span class="sourceLineNo">1046</span>        // Debug<a name="line.1046"></a>
<span class="sourceLineNo">1047</span>        if (this.log.isTraceEnabled())<a name="line.1047"></a>
<span class="sourceLineNo">1048</span>            this.trace("cleaning up transaction " + tx);<a name="line.1048"></a>
<span class="sourceLineNo">1049</span><a name="line.1049"></a>
<span class="sourceLineNo">1050</span>        // Do any per-role cleanups<a name="line.1050"></a>
<span class="sourceLineNo">1051</span>        if (this.role != null)<a name="line.1051"></a>
<span class="sourceLineNo">1052</span>            this.role.cleanupForTransaction(tx);<a name="line.1052"></a>
<span class="sourceLineNo">1053</span><a name="line.1053"></a>
<span class="sourceLineNo">1054</span>        // Close transaction's snapshot<a name="line.1054"></a>
<span class="sourceLineNo">1055</span>        tx.closeSnapshot();<a name="line.1055"></a>
<span class="sourceLineNo">1056</span><a name="line.1056"></a>
<span class="sourceLineNo">1057</span>        // Cancel commit timer<a name="line.1057"></a>
<span class="sourceLineNo">1058</span>        final Timer commitTimer = tx.getCommitTimer();<a name="line.1058"></a>
<span class="sourceLineNo">1059</span>        if (commitTimer != null)<a name="line.1059"></a>
<span class="sourceLineNo">1060</span>            commitTimer.cancel();<a name="line.1060"></a>
<span class="sourceLineNo">1061</span><a name="line.1061"></a>
<span class="sourceLineNo">1062</span>        // Remove from open transactions set<a name="line.1062"></a>
<span class="sourceLineNo">1063</span>        this.openTransactions.remove(tx.getTxId());<a name="line.1063"></a>
<span class="sourceLineNo">1064</span><a name="line.1064"></a>
<span class="sourceLineNo">1065</span>        // Transition to CLOSED<a name="line.1065"></a>
<span class="sourceLineNo">1066</span>        tx.setState(TxState.CLOSED);<a name="line.1066"></a>
<span class="sourceLineNo">1067</span><a name="line.1067"></a>
<span class="sourceLineNo">1068</span>        // Notify waiting thread if doing shutdown<a name="line.1068"></a>
<span class="sourceLineNo">1069</span>        if (this.shuttingDown)<a name="line.1069"></a>
<span class="sourceLineNo">1070</span>            this.notify();<a name="line.1070"></a>
<span class="sourceLineNo">1071</span>    }<a name="line.1071"></a>
<span class="sourceLineNo">1072</span><a name="line.1072"></a>
<span class="sourceLineNo">1073</span>    // Mark a transaction as having succeeded<a name="line.1073"></a>
<span class="sourceLineNo">1074</span>    void succeed(RaftKVTransaction tx) {<a name="line.1074"></a>
<span class="sourceLineNo">1075</span><a name="line.1075"></a>
<span class="sourceLineNo">1076</span>        // Sanity check<a name="line.1076"></a>
<span class="sourceLineNo">1077</span>        assert Thread.holdsLock(this);<a name="line.1077"></a>
<span class="sourceLineNo">1078</span>        assert this.role != null;<a name="line.1078"></a>
<span class="sourceLineNo">1079</span>        if (tx.getState().equals(TxState.COMPLETED) || tx.getState().equals(TxState.CLOSED))<a name="line.1079"></a>
<span class="sourceLineNo">1080</span>            return;<a name="line.1080"></a>
<span class="sourceLineNo">1081</span><a name="line.1081"></a>
<span class="sourceLineNo">1082</span>        // Succeed transaction<a name="line.1082"></a>
<span class="sourceLineNo">1083</span>        if (this.log.isDebugEnabled())<a name="line.1083"></a>
<span class="sourceLineNo">1084</span>            this.debug("successfully committed " + tx);<a name="line.1084"></a>
<span class="sourceLineNo">1085</span>        tx.getCommitFuture().succeed();<a name="line.1085"></a>
<span class="sourceLineNo">1086</span>        tx.setState(TxState.COMPLETED);<a name="line.1086"></a>
<span class="sourceLineNo">1087</span>        this.role.cleanupForTransaction(tx);<a name="line.1087"></a>
<span class="sourceLineNo">1088</span>    }<a name="line.1088"></a>
<span class="sourceLineNo">1089</span><a name="line.1089"></a>
<span class="sourceLineNo">1090</span>    // Mark a transaction as having failed<a name="line.1090"></a>
<span class="sourceLineNo">1091</span>    void fail(RaftKVTransaction tx, Exception e) {<a name="line.1091"></a>
<span class="sourceLineNo">1092</span><a name="line.1092"></a>
<span class="sourceLineNo">1093</span>        // Sanity check<a name="line.1093"></a>
<span class="sourceLineNo">1094</span>        assert Thread.holdsLock(this);<a name="line.1094"></a>
<span class="sourceLineNo">1095</span>        assert this.role != null;<a name="line.1095"></a>
<span class="sourceLineNo">1096</span>        if (tx.getState().equals(TxState.COMPLETED) || tx.getState().equals(TxState.CLOSED))<a name="line.1096"></a>
<span class="sourceLineNo">1097</span>            return;<a name="line.1097"></a>
<span class="sourceLineNo">1098</span><a name="line.1098"></a>
<span class="sourceLineNo">1099</span>        // Fail transaction<a name="line.1099"></a>
<span class="sourceLineNo">1100</span>        if (this.log.isDebugEnabled())<a name="line.1100"></a>
<span class="sourceLineNo">1101</span>            this.debug("failed transaction " + tx + ": " + e);<a name="line.1101"></a>
<span class="sourceLineNo">1102</span>        tx.getCommitFuture().fail(e);<a name="line.1102"></a>
<span class="sourceLineNo">1103</span>        tx.setState(TxState.COMPLETED);<a name="line.1103"></a>
<span class="sourceLineNo">1104</span>        this.role.cleanupForTransaction(tx);<a name="line.1104"></a>
<span class="sourceLineNo">1105</span>    }<a name="line.1105"></a>
<span class="sourceLineNo">1106</span><a name="line.1106"></a>
<span class="sourceLineNo">1107</span>// DataView<a name="line.1107"></a>
<span class="sourceLineNo">1108</span><a name="line.1108"></a>
<span class="sourceLineNo">1109</span>    /**<a name="line.1109"></a>
<span class="sourceLineNo">1110</span>     * A view of the database based on the most recent log entry, if any, otherwise directly on the committed key/value store.<a name="line.1110"></a>
<span class="sourceLineNo">1111</span>     * Caller is responsible for eventually closing the snapshot.<a name="line.1111"></a>
<span class="sourceLineNo">1112</span>     */<a name="line.1112"></a>
<span class="sourceLineNo">1113</span>    private class MostRecentView {<a name="line.1113"></a>
<span class="sourceLineNo">1114</span><a name="line.1114"></a>
<span class="sourceLineNo">1115</span>        private final CloseableKVStore snapshot;<a name="line.1115"></a>
<span class="sourceLineNo">1116</span>        private final MutableView view;<a name="line.1116"></a>
<span class="sourceLineNo">1117</span><a name="line.1117"></a>
<span class="sourceLineNo">1118</span>        public MostRecentView() {<a name="line.1118"></a>
<span class="sourceLineNo">1119</span>            assert Thread.holdsLock(RaftKVDatabase.this);<a name="line.1119"></a>
<span class="sourceLineNo">1120</span><a name="line.1120"></a>
<span class="sourceLineNo">1121</span>            // Grab a snapshot of the key/value store<a name="line.1121"></a>
<span class="sourceLineNo">1122</span>            this.snapshot = RaftKVDatabase.this.kv.snapshot();<a name="line.1122"></a>
<span class="sourceLineNo">1123</span><a name="line.1123"></a>
<span class="sourceLineNo">1124</span>            // Create a view of just the state machine keys and values and successively layer unapplied log entries<a name="line.1124"></a>
<span class="sourceLineNo">1125</span>            KVStore kview = PrefixKVStore.create(snapshot, STATE_MACHINE_PREFIX);<a name="line.1125"></a>
<span class="sourceLineNo">1126</span>            for (LogEntry logEntry : RaftKVDatabase.this.raftLog) {<a name="line.1126"></a>
<span class="sourceLineNo">1127</span>                final Writes writes = logEntry.getWrites();<a name="line.1127"></a>
<span class="sourceLineNo">1128</span>                if (!writes.isEmpty())<a name="line.1128"></a>
<span class="sourceLineNo">1129</span>                    kview = new MutableView(kview, null, logEntry.getWrites());<a name="line.1129"></a>
<span class="sourceLineNo">1130</span>            }<a name="line.1130"></a>
<span class="sourceLineNo">1131</span><a name="line.1131"></a>
<span class="sourceLineNo">1132</span>            // Finalize<a name="line.1132"></a>
<span class="sourceLineNo">1133</span>            this.view = new MutableView(kview);<a name="line.1133"></a>
<span class="sourceLineNo">1134</span>        }<a name="line.1134"></a>
<span class="sourceLineNo">1135</span><a name="line.1135"></a>
<span class="sourceLineNo">1136</span>        public CloseableKVStore getSnapshot() {<a name="line.1136"></a>
<span class="sourceLineNo">1137</span>            return this.snapshot;<a name="line.1137"></a>
<span class="sourceLineNo">1138</span>        }<a name="line.1138"></a>
<span class="sourceLineNo">1139</span><a name="line.1139"></a>
<span class="sourceLineNo">1140</span>        public MutableView getView() {<a name="line.1140"></a>
<span class="sourceLineNo">1141</span>            return this.view;<a name="line.1141"></a>
<span class="sourceLineNo">1142</span>        }<a name="line.1142"></a>
<span class="sourceLineNo">1143</span>    }<a name="line.1143"></a>
<span class="sourceLineNo">1144</span><a name="line.1144"></a>
<span class="sourceLineNo">1145</span>// Service<a name="line.1145"></a>
<span class="sourceLineNo">1146</span><a name="line.1146"></a>
<span class="sourceLineNo">1147</span>    /**<a name="line.1147"></a>
<span class="sourceLineNo">1148</span>     * Request service to be invoked after the current service (if any) completes.<a name="line.1148"></a>
<span class="sourceLineNo">1149</span>     *<a name="line.1149"></a>
<span class="sourceLineNo">1150</span>     * &lt;p&gt;<a name="line.1150"></a>
<span class="sourceLineNo">1151</span>     * If {@code service} has an associated {@link Role}, and that {@link Role} is no longer active<a name="line.1151"></a>
<span class="sourceLineNo">1152</span>     * when the service is handled, nothing will be done.<a name="line.1152"></a>
<span class="sourceLineNo">1153</span>     *<a name="line.1153"></a>
<span class="sourceLineNo">1154</span>     * @param service the service to perform<a name="line.1154"></a>
<span class="sourceLineNo">1155</span>     */<a name="line.1155"></a>
<span class="sourceLineNo">1156</span>    private void requestService(Service service) {<a name="line.1156"></a>
<span class="sourceLineNo">1157</span>        assert Thread.holdsLock(this);<a name="line.1157"></a>
<span class="sourceLineNo">1158</span>        if (!this.pendingService.add(service))<a name="line.1158"></a>
<span class="sourceLineNo">1159</span>            return;<a name="line.1159"></a>
<span class="sourceLineNo">1160</span>        if (this.performingService)<a name="line.1160"></a>
<span class="sourceLineNo">1161</span>            return;<a name="line.1161"></a>
<span class="sourceLineNo">1162</span>        try {<a name="line.1162"></a>
<span class="sourceLineNo">1163</span>            this.executor.submit(new ErrorLoggingRunnable() {<a name="line.1163"></a>
<span class="sourceLineNo">1164</span>                @Override<a name="line.1164"></a>
<span class="sourceLineNo">1165</span>                protected void doRun() {<a name="line.1165"></a>
<span class="sourceLineNo">1166</span>                    RaftKVDatabase.this.handlePendingService();<a name="line.1166"></a>
<span class="sourceLineNo">1167</span>                }<a name="line.1167"></a>
<span class="sourceLineNo">1168</span>            });<a name="line.1168"></a>
<span class="sourceLineNo">1169</span>        } catch (RejectedExecutionException e) {<a name="line.1169"></a>
<span class="sourceLineNo">1170</span>            if (!this.shuttingDown)<a name="line.1170"></a>
<span class="sourceLineNo">1171</span>                this.warn("executor task rejected, skipping", e);<a name="line.1171"></a>
<span class="sourceLineNo">1172</span>        }<a name="line.1172"></a>
<span class="sourceLineNo">1173</span>    }<a name="line.1173"></a>
<span class="sourceLineNo">1174</span><a name="line.1174"></a>
<span class="sourceLineNo">1175</span>    // Performs pending service requests (do not invoke directly)<a name="line.1175"></a>
<span class="sourceLineNo">1176</span>    private synchronized void handlePendingService() {<a name="line.1176"></a>
<span class="sourceLineNo">1177</span><a name="line.1177"></a>
<span class="sourceLineNo">1178</span>        // Sanity check<a name="line.1178"></a>
<span class="sourceLineNo">1179</span>        assert this.checkState();<a name="line.1179"></a>
<span class="sourceLineNo">1180</span>        assert Thread.currentThread() instanceof ServiceThread;<a name="line.1180"></a>
<span class="sourceLineNo">1181</span>        if (this.role == null)<a name="line.1181"></a>
<span class="sourceLineNo">1182</span>            return;<a name="line.1182"></a>
<span class="sourceLineNo">1183</span><a name="line.1183"></a>
<span class="sourceLineNo">1184</span>        // While there is work to do, do it<a name="line.1184"></a>
<span class="sourceLineNo">1185</span>        this.performingService = true;<a name="line.1185"></a>
<span class="sourceLineNo">1186</span>        try {<a name="line.1186"></a>
<span class="sourceLineNo">1187</span>            while (!this.pendingService.isEmpty()) {<a name="line.1187"></a>
<span class="sourceLineNo">1188</span>                final Iterator&lt;Service&gt; i = this.pendingService.iterator();<a name="line.1188"></a>
<span class="sourceLineNo">1189</span>                final Service service = i.next();<a name="line.1189"></a>
<span class="sourceLineNo">1190</span>                i.remove();<a name="line.1190"></a>
<span class="sourceLineNo">1191</span>                assert service.getRole() == null || service.getRole() == this.role;<a name="line.1191"></a>
<span class="sourceLineNo">1192</span>                if (this.log.isTraceEnabled())<a name="line.1192"></a>
<span class="sourceLineNo">1193</span>                    this.trace("SERVICE [" + service + "] in " + this.role);<a name="line.1193"></a>
<span class="sourceLineNo">1194</span>                service.run();<a name="line.1194"></a>
<span class="sourceLineNo">1195</span>            }<a name="line.1195"></a>
<span class="sourceLineNo">1196</span>        } finally {<a name="line.1196"></a>
<span class="sourceLineNo">1197</span>            this.performingService = false;<a name="line.1197"></a>
<span class="sourceLineNo">1198</span>        }<a name="line.1198"></a>
<span class="sourceLineNo">1199</span>    }<a name="line.1199"></a>
<span class="sourceLineNo">1200</span><a name="line.1200"></a>
<span class="sourceLineNo">1201</span>    private class ServiceThread extends Thread {<a name="line.1201"></a>
<span class="sourceLineNo">1202</span>        ServiceThread(Runnable action) {<a name="line.1202"></a>
<span class="sourceLineNo">1203</span>            super(action);<a name="line.1203"></a>
<span class="sourceLineNo">1204</span>            this.setName(RaftKVDatabase.this + " Service");<a name="line.1204"></a>
<span class="sourceLineNo">1205</span>        }<a name="line.1205"></a>
<span class="sourceLineNo">1206</span>    }<a name="line.1206"></a>
<span class="sourceLineNo">1207</span><a name="line.1207"></a>
<span class="sourceLineNo">1208</span>    private abstract static class Service implements Runnable {<a name="line.1208"></a>
<span class="sourceLineNo">1209</span><a name="line.1209"></a>
<span class="sourceLineNo">1210</span>        private final Role role;<a name="line.1210"></a>
<span class="sourceLineNo">1211</span>        private final String desc;<a name="line.1211"></a>
<span class="sourceLineNo">1212</span><a name="line.1212"></a>
<span class="sourceLineNo">1213</span>        protected Service(Role role, String desc) {<a name="line.1213"></a>
<span class="sourceLineNo">1214</span>            this.role = role;<a name="line.1214"></a>
<span class="sourceLineNo">1215</span>            this.desc = desc;<a name="line.1215"></a>
<span class="sourceLineNo">1216</span>        }<a name="line.1216"></a>
<span class="sourceLineNo">1217</span><a name="line.1217"></a>
<span class="sourceLineNo">1218</span>        public Role getRole() {<a name="line.1218"></a>
<span class="sourceLineNo">1219</span>            return this.role;<a name="line.1219"></a>
<span class="sourceLineNo">1220</span>        }<a name="line.1220"></a>
<span class="sourceLineNo">1221</span><a name="line.1221"></a>
<span class="sourceLineNo">1222</span>        @Override<a name="line.1222"></a>
<span class="sourceLineNo">1223</span>        public String toString() {<a name="line.1223"></a>
<span class="sourceLineNo">1224</span>            return this.desc;<a name="line.1224"></a>
<span class="sourceLineNo">1225</span>        }<a name="line.1225"></a>
<span class="sourceLineNo">1226</span>    }<a name="line.1226"></a>
<span class="sourceLineNo">1227</span><a name="line.1227"></a>
<span class="sourceLineNo">1228</span>// Timer<a name="line.1228"></a>
<span class="sourceLineNo">1229</span><a name="line.1229"></a>
<span class="sourceLineNo">1230</span>    /**<a name="line.1230"></a>
<span class="sourceLineNo">1231</span>     * One shot timer that {@linkplain #requestService requests} a {@link Service} on expiration.<a name="line.1231"></a>
<span class="sourceLineNo">1232</span>     */<a name="line.1232"></a>
<span class="sourceLineNo">1233</span>    class Timer {<a name="line.1233"></a>
<span class="sourceLineNo">1234</span><a name="line.1234"></a>
<span class="sourceLineNo">1235</span>        private final Logger log = RaftKVDatabase.this.log;<a name="line.1235"></a>
<span class="sourceLineNo">1236</span>        private final String name;<a name="line.1236"></a>
<span class="sourceLineNo">1237</span>        private final Service service;<a name="line.1237"></a>
<span class="sourceLineNo">1238</span>        private ScheduledFuture&lt;?&gt; future;<a name="line.1238"></a>
<span class="sourceLineNo">1239</span>        private PendingTimeout pendingTimeout;                  // non-null IFF timeout has not been handled yet<a name="line.1239"></a>
<span class="sourceLineNo">1240</span>        private Timestamp timeoutDeadline;<a name="line.1240"></a>
<span class="sourceLineNo">1241</span><a name="line.1241"></a>
<span class="sourceLineNo">1242</span>        public Timer(String name, Service service) {<a name="line.1242"></a>
<span class="sourceLineNo">1243</span>            this.name = name;<a name="line.1243"></a>
<span class="sourceLineNo">1244</span>            this.service = service;<a name="line.1244"></a>
<span class="sourceLineNo">1245</span>        }<a name="line.1245"></a>
<span class="sourceLineNo">1246</span><a name="line.1246"></a>
<span class="sourceLineNo">1247</span>        /**<a name="line.1247"></a>
<span class="sourceLineNo">1248</span>         * Stop timer if running.<a name="line.1248"></a>
<span class="sourceLineNo">1249</span>         *<a name="line.1249"></a>
<span class="sourceLineNo">1250</span>         * @throws IllegalStateException if the lock object is not locked<a name="line.1250"></a>
<span class="sourceLineNo">1251</span>         */<a name="line.1251"></a>
<span class="sourceLineNo">1252</span>        public void cancel() {<a name="line.1252"></a>
<span class="sourceLineNo">1253</span><a name="line.1253"></a>
<span class="sourceLineNo">1254</span>            // Sanity check<a name="line.1254"></a>
<span class="sourceLineNo">1255</span>            assert Thread.holdsLock(RaftKVDatabase.this);<a name="line.1255"></a>
<span class="sourceLineNo">1256</span><a name="line.1256"></a>
<span class="sourceLineNo">1257</span>            // Cancel existing timer, if any<a name="line.1257"></a>
<span class="sourceLineNo">1258</span>            if (this.future != null) {<a name="line.1258"></a>
<span class="sourceLineNo">1259</span>                this.future.cancel(false);<a name="line.1259"></a>
<span class="sourceLineNo">1260</span>                this.future = null;<a name="line.1260"></a>
<span class="sourceLineNo">1261</span>            }<a name="line.1261"></a>
<span class="sourceLineNo">1262</span><a name="line.1262"></a>
<span class="sourceLineNo">1263</span>            // Ensure the previously scheduled action does nothing if case we lose the cancel() race condition<a name="line.1263"></a>
<span class="sourceLineNo">1264</span>            this.pendingTimeout = null;<a name="line.1264"></a>
<span class="sourceLineNo">1265</span>            this.timeoutDeadline = null;<a name="line.1265"></a>
<span class="sourceLineNo">1266</span>        }<a name="line.1266"></a>
<span class="sourceLineNo">1267</span><a name="line.1267"></a>
<span class="sourceLineNo">1268</span>        /**<a name="line.1268"></a>
<span class="sourceLineNo">1269</span>         * (Re)schedule this timer. Discards any previously scheduled timeout.<a name="line.1269"></a>
<span class="sourceLineNo">1270</span>         *<a name="line.1270"></a>
<span class="sourceLineNo">1271</span>         * @param delay delay before expiration in milliseonds<a name="line.1271"></a>
<span class="sourceLineNo">1272</span>         * @return true if restarted, false if executor rejected the task<a name="line.1272"></a>
<span class="sourceLineNo">1273</span>         * @throws IllegalStateException if the lock object is not locked<a name="line.1273"></a>
<span class="sourceLineNo">1274</span>         */<a name="line.1274"></a>
<span class="sourceLineNo">1275</span>        public void timeoutAfter(int delay) {<a name="line.1275"></a>
<span class="sourceLineNo">1276</span><a name="line.1276"></a>
<span class="sourceLineNo">1277</span>            // Sanity check<a name="line.1277"></a>
<span class="sourceLineNo">1278</span>            assert Thread.holdsLock(RaftKVDatabase.this);<a name="line.1278"></a>
<span class="sourceLineNo">1279</span>            Preconditions.checkArgument(delay &gt;= 0, "delay &lt; 0");<a name="line.1279"></a>
<span class="sourceLineNo">1280</span><a name="line.1280"></a>
<span class="sourceLineNo">1281</span>            // Cancel existing timeout action, if any<a name="line.1281"></a>
<span class="sourceLineNo">1282</span>            this.cancel();<a name="line.1282"></a>
<span class="sourceLineNo">1283</span>            assert this.future == null;<a name="line.1283"></a>
<span class="sourceLineNo">1284</span>            assert this.pendingTimeout == null;<a name="line.1284"></a>
<span class="sourceLineNo">1285</span>            assert this.timeoutDeadline == null;<a name="line.1285"></a>
<span class="sourceLineNo">1286</span><a name="line.1286"></a>
<span class="sourceLineNo">1287</span>            // Reschedule new timeout action<a name="line.1287"></a>
<span class="sourceLineNo">1288</span>            this.timeoutDeadline = new Timestamp().offset(delay);<a name="line.1288"></a>
<span class="sourceLineNo">1289</span>            if (this.log.isTraceEnabled()) {<a name="line.1289"></a>
<span class="sourceLineNo">1290</span>                RaftKVDatabase.this.trace("rescheduling " + this.name + " for " + this.timeoutDeadline<a name="line.1290"></a>
<span class="sourceLineNo">1291</span>                  + " (" + delay + "ms from now)");<a name="line.1291"></a>
<span class="sourceLineNo">1292</span>            }<a name="line.1292"></a>
<span class="sourceLineNo">1293</span>            this.pendingTimeout = new PendingTimeout();<a name="line.1293"></a>
<span class="sourceLineNo">1294</span>            try {<a name="line.1294"></a>
<span class="sourceLineNo">1295</span>                this.future = RaftKVDatabase.this.executor.schedule(this.pendingTimeout, delay, TimeUnit.MILLISECONDS);<a name="line.1295"></a>
<span class="sourceLineNo">1296</span>            } catch (RejectedExecutionException e) {<a name="line.1296"></a>
<span class="sourceLineNo">1297</span>                if (!RaftKVDatabase.this.shuttingDown)<a name="line.1297"></a>
<span class="sourceLineNo">1298</span>                    RaftKVDatabase.this.warn("can't restart timer", e);<a name="line.1298"></a>
<span class="sourceLineNo">1299</span>            }<a name="line.1299"></a>
<span class="sourceLineNo">1300</span>        }<a name="line.1300"></a>
<span class="sourceLineNo">1301</span><a name="line.1301"></a>
<span class="sourceLineNo">1302</span>        /**<a name="line.1302"></a>
<span class="sourceLineNo">1303</span>         * Force timer to expire immediately.<a name="line.1303"></a>
<span class="sourceLineNo">1304</span>         */<a name="line.1304"></a>
<span class="sourceLineNo">1305</span>        public void timeoutNow() {<a name="line.1305"></a>
<span class="sourceLineNo">1306</span>            this.timeoutAfter(0);<a name="line.1306"></a>
<span class="sourceLineNo">1307</span>        }<a name="line.1307"></a>
<span class="sourceLineNo">1308</span><a name="line.1308"></a>
<span class="sourceLineNo">1309</span>        /**<a name="line.1309"></a>
<span class="sourceLineNo">1310</span>         * Determine if this timer has expired and requires service handling, and reset it if so.<a name="line.1310"></a>
<span class="sourceLineNo">1311</span>         *<a name="line.1311"></a>
<span class="sourceLineNo">1312</span>         * &lt;p&gt;<a name="line.1312"></a>
<span class="sourceLineNo">1313</span>         * If this timer is not running, has not yet expired, or has previously expired and this method was already<a name="line.1313"></a>
<span class="sourceLineNo">1314</span>         * thereafter invoked, false is returned. Otherwise, true is returned, this timer is {@link #cancel}ed (if necessary),<a name="line.1314"></a>
<span class="sourceLineNo">1315</span>         * and the caller is expected to handle the implied service need.<a name="line.1315"></a>
<span class="sourceLineNo">1316</span>         *<a name="line.1316"></a>
<span class="sourceLineNo">1317</span>         * @return true if timer needs handling, false otherwise<a name="line.1317"></a>
<span class="sourceLineNo">1318</span>         */<a name="line.1318"></a>
<span class="sourceLineNo">1319</span>        public boolean pollForTimeout() {<a name="line.1319"></a>
<span class="sourceLineNo">1320</span><a name="line.1320"></a>
<span class="sourceLineNo">1321</span>            // Sanity check<a name="line.1321"></a>
<span class="sourceLineNo">1322</span>            assert Thread.holdsLock(RaftKVDatabase.this);<a name="line.1322"></a>
<span class="sourceLineNo">1323</span><a name="line.1323"></a>
<span class="sourceLineNo">1324</span>            // Has timer expired?<a name="line.1324"></a>
<span class="sourceLineNo">1325</span>            if (this.pendingTimeout == null || !this.timeoutDeadline.hasOccurred())<a name="line.1325"></a>
<span class="sourceLineNo">1326</span>                return false;<a name="line.1326"></a>
<span class="sourceLineNo">1327</span><a name="line.1327"></a>
<span class="sourceLineNo">1328</span>            // Yes, timer requires service<a name="line.1328"></a>
<span class="sourceLineNo">1329</span>            if (Timer.this.log.isTraceEnabled())<a name="line.1329"></a>
<span class="sourceLineNo">1330</span>                RaftKVDatabase.this.trace(Timer.this.name + " expired " + -this.timeoutDeadline.offsetFromNow() + "ms ago");<a name="line.1330"></a>
<span class="sourceLineNo">1331</span>            this.cancel();<a name="line.1331"></a>
<span class="sourceLineNo">1332</span>            return true;<a name="line.1332"></a>
<span class="sourceLineNo">1333</span>        }<a name="line.1333"></a>
<span class="sourceLineNo">1334</span><a name="line.1334"></a>
<span class="sourceLineNo">1335</span>        /**<a name="line.1335"></a>
<span class="sourceLineNo">1336</span>         * Determine if this timer is running, i.e., will expire or has expired but<a name="line.1336"></a>
<span class="sourceLineNo">1337</span>         * {@link #pollForTimeout} has not been invoked yet.<a name="line.1337"></a>
<span class="sourceLineNo">1338</span>         */<a name="line.1338"></a>
<span class="sourceLineNo">1339</span>        public boolean isRunning() {<a name="line.1339"></a>
<span class="sourceLineNo">1340</span>            return this.pendingTimeout != null;<a name="line.1340"></a>
<span class="sourceLineNo">1341</span>        }<a name="line.1341"></a>
<span class="sourceLineNo">1342</span><a name="line.1342"></a>
<span class="sourceLineNo">1343</span>        private class PendingTimeout extends ErrorLoggingRunnable {<a name="line.1343"></a>
<span class="sourceLineNo">1344</span><a name="line.1344"></a>
<span class="sourceLineNo">1345</span>            @Override<a name="line.1345"></a>
<span class="sourceLineNo">1346</span>            protected void doRun() {<a name="line.1346"></a>
<span class="sourceLineNo">1347</span>                synchronized (RaftKVDatabase.this) {<a name="line.1347"></a>
<span class="sourceLineNo">1348</span><a name="line.1348"></a>
<span class="sourceLineNo">1349</span>                    // Avoid cancel() race condition<a name="line.1349"></a>
<span class="sourceLineNo">1350</span>                    if (Timer.this.pendingTimeout != this)<a name="line.1350"></a>
<span class="sourceLineNo">1351</span>                        return;<a name="line.1351"></a>
<span class="sourceLineNo">1352</span><a name="line.1352"></a>
<span class="sourceLineNo">1353</span>                    // Trigger service<a name="line.1353"></a>
<span class="sourceLineNo">1354</span>                    RaftKVDatabase.this.requestService(Timer.this.service);<a name="line.1354"></a>
<span class="sourceLineNo">1355</span>                }<a name="line.1355"></a>
<span class="sourceLineNo">1356</span>            }<a name="line.1356"></a>
<span class="sourceLineNo">1357</span>        }<a name="line.1357"></a>
<span class="sourceLineNo">1358</span>    }<a name="line.1358"></a>
<span class="sourceLineNo">1359</span><a name="line.1359"></a>
<span class="sourceLineNo">1360</span>// Raft stuff<a name="line.1360"></a>
<span class="sourceLineNo">1361</span><a name="line.1361"></a>
<span class="sourceLineNo">1362</span>    /**<a name="line.1362"></a>
<span class="sourceLineNo">1363</span>     * Reset the persisted state machine to its initial state.<a name="line.1363"></a>
<span class="sourceLineNo">1364</span>     */<a name="line.1364"></a>
<span class="sourceLineNo">1365</span>    private boolean resetStateMachine(boolean initialize) {<a name="line.1365"></a>
<span class="sourceLineNo">1366</span><a name="line.1366"></a>
<span class="sourceLineNo">1367</span>        // Sanity check<a name="line.1367"></a>
<span class="sourceLineNo">1368</span>        assert Thread.holdsLock(this);<a name="line.1368"></a>
<span class="sourceLineNo">1369</span>        if (this.log.isDebugEnabled())<a name="line.1369"></a>
<span class="sourceLineNo">1370</span>            this.debug("resetting state machine");<a name="line.1370"></a>
<span class="sourceLineNo">1371</span><a name="line.1371"></a>
<span class="sourceLineNo">1372</span>        // Set invalid values while we make non-atomic changes, in case we crash in the middle<a name="line.1372"></a>
<span class="sourceLineNo">1373</span>        if (!this.recordLastApplied(-1, -1, null))<a name="line.1373"></a>
<span class="sourceLineNo">1374</span>            return false;<a name="line.1374"></a>
<span class="sourceLineNo">1375</span><a name="line.1375"></a>
<span class="sourceLineNo">1376</span>        // Delete all key/value pairs<a name="line.1376"></a>
<span class="sourceLineNo">1377</span>        this.kv.removeRange(STATE_MACHINE_PREFIX, ByteUtil.getKeyAfterPrefix(STATE_MACHINE_PREFIX));<a name="line.1377"></a>
<span class="sourceLineNo">1378</span>        assert !this.kv.getRange(STATE_MACHINE_PREFIX, ByteUtil.getKeyAfterPrefix(STATE_MACHINE_PREFIX), false).hasNext();<a name="line.1378"></a>
<span class="sourceLineNo">1379</span><a name="line.1379"></a>
<span class="sourceLineNo">1380</span>        // Delete all log files<a name="line.1380"></a>
<span class="sourceLineNo">1381</span>        this.raftLog.clear();<a name="line.1381"></a>
<span class="sourceLineNo">1382</span>        for (File file : this.logDir.listFiles()) {<a name="line.1382"></a>
<span class="sourceLineNo">1383</span>            if (LogEntry.LOG_FILE_PATTERN.matcher(file.getName()).matches()) {<a name="line.1383"></a>
<span class="sourceLineNo">1384</span>                if (!file.delete())<a name="line.1384"></a>
<span class="sourceLineNo">1385</span>                    this.error("failed to delete log file " + file);<a name="line.1385"></a>
<span class="sourceLineNo">1386</span>            }<a name="line.1386"></a>
<span class="sourceLineNo">1387</span>        }<a name="line.1387"></a>
<span class="sourceLineNo">1388</span><a name="line.1388"></a>
<span class="sourceLineNo">1389</span>        // Optionally finish intialization<a name="line.1389"></a>
<span class="sourceLineNo">1390</span>        if (initialize &amp;&amp; !this.recordLastApplied(0, 0, null))<a name="line.1390"></a>
<span class="sourceLineNo">1391</span>            return false;<a name="line.1391"></a>
<span class="sourceLineNo">1392</span><a name="line.1392"></a>
<span class="sourceLineNo">1393</span>        // Done<a name="line.1393"></a>
<span class="sourceLineNo">1394</span>        if (this.log.isDebugEnabled())<a name="line.1394"></a>
<span class="sourceLineNo">1395</span>            this.debug("done resetting state machine");<a name="line.1395"></a>
<span class="sourceLineNo">1396</span>        return true;<a name="line.1396"></a>
<span class="sourceLineNo">1397</span>    }<a name="line.1397"></a>
<span class="sourceLineNo">1398</span><a name="line.1398"></a>
<span class="sourceLineNo">1399</span>    /**<a name="line.1399"></a>
<span class="sourceLineNo">1400</span>     * Record the last applied term, index, and configuration in the persistent store.<a name="line.1400"></a>
<span class="sourceLineNo">1401</span>     */<a name="line.1401"></a>
<span class="sourceLineNo">1402</span>    private boolean recordLastApplied(long term, long index, Map&lt;String, String&gt; config) {<a name="line.1402"></a>
<span class="sourceLineNo">1403</span><a name="line.1403"></a>
<span class="sourceLineNo">1404</span>        // Sanity check<a name="line.1404"></a>
<span class="sourceLineNo">1405</span>        assert Thread.holdsLock(this);<a name="line.1405"></a>
<span class="sourceLineNo">1406</span>        if (this.log.isTraceEnabled())<a name="line.1406"></a>
<span class="sourceLineNo">1407</span>            this.trace("updating state machine last applied to " + term + "/" + index + " with config " + config);<a name="line.1407"></a>
<span class="sourceLineNo">1408</span>        if (config == null)<a name="line.1408"></a>
<span class="sourceLineNo">1409</span>            config = new HashMap&lt;String, String&gt;(0);<a name="line.1409"></a>
<span class="sourceLineNo">1410</span><a name="line.1410"></a>
<span class="sourceLineNo">1411</span>        // Prepare updates<a name="line.1411"></a>
<span class="sourceLineNo">1412</span>        final Writes writes = new Writes();<a name="line.1412"></a>
<span class="sourceLineNo">1413</span>        writes.getPuts().put(LAST_APPLIED_TERM_KEY, LongEncoder.encode(term));<a name="line.1413"></a>
<span class="sourceLineNo">1414</span>        writes.getPuts().put(LAST_APPLIED_INDEX_KEY, LongEncoder.encode(index));<a name="line.1414"></a>
<span class="sourceLineNo">1415</span>        writes.getPuts().put(LAST_APPLIED_CONFIG_KEY, this.encodeConfig(config));<a name="line.1415"></a>
<span class="sourceLineNo">1416</span><a name="line.1416"></a>
<span class="sourceLineNo">1417</span>        // Update persistent store<a name="line.1417"></a>
<span class="sourceLineNo">1418</span>        try {<a name="line.1418"></a>
<span class="sourceLineNo">1419</span>            this.kv.mutate(writes, true);<a name="line.1419"></a>
<span class="sourceLineNo">1420</span>        } catch (Exception e) {<a name="line.1420"></a>
<span class="sourceLineNo">1421</span>            this.error("error updating key/value store term/index to " + term + "/" + index, e);<a name="line.1421"></a>
<span class="sourceLineNo">1422</span>            return false;<a name="line.1422"></a>
<span class="sourceLineNo">1423</span>        }<a name="line.1423"></a>
<span class="sourceLineNo">1424</span><a name="line.1424"></a>
<span class="sourceLineNo">1425</span>        // Update in-memory copy<a name="line.1425"></a>
<span class="sourceLineNo">1426</span>        this.lastAppliedTerm = Math.max(term, 0);<a name="line.1426"></a>
<span class="sourceLineNo">1427</span>        this.lastAppliedIndex = Math.max(index, 0);<a name="line.1427"></a>
<span class="sourceLineNo">1428</span>        this.lastAppliedConfig = config;<a name="line.1428"></a>
<span class="sourceLineNo">1429</span>        this.commitIndex = this.lastAppliedIndex;<a name="line.1429"></a>
<span class="sourceLineNo">1430</span>        this.currentConfig = this.buildCurrentConfig();<a name="line.1430"></a>
<span class="sourceLineNo">1431</span>        if (term &gt;= 0 &amp;&amp; index &gt;= 0)<a name="line.1431"></a>
<span class="sourceLineNo">1432</span>            this.info("new current configuration: " + this.currentConfig);<a name="line.1432"></a>
<span class="sourceLineNo">1433</span>        return true;<a name="line.1433"></a>
<span class="sourceLineNo">1434</span>    }<a name="line.1434"></a>
<span class="sourceLineNo">1435</span><a name="line.1435"></a>
<span class="sourceLineNo">1436</span>    /**<a name="line.1436"></a>
<span class="sourceLineNo">1437</span>     * Update and persist a new current term.<a name="line.1437"></a>
<span class="sourceLineNo">1438</span>     */<a name="line.1438"></a>
<span class="sourceLineNo">1439</span>    private boolean advanceTerm(long newTerm) {<a name="line.1439"></a>
<span class="sourceLineNo">1440</span><a name="line.1440"></a>
<span class="sourceLineNo">1441</span>        // Sanity check<a name="line.1441"></a>
<span class="sourceLineNo">1442</span>        assert Thread.holdsLock(this);<a name="line.1442"></a>
<span class="sourceLineNo">1443</span>        assert newTerm &gt; this.currentTerm;<a name="line.1443"></a>
<span class="sourceLineNo">1444</span>        this.info("advancing current term from " + this.currentTerm + " -&gt; " + newTerm);<a name="line.1444"></a>
<span class="sourceLineNo">1445</span><a name="line.1445"></a>
<span class="sourceLineNo">1446</span>        // Update persistent store<a name="line.1446"></a>
<span class="sourceLineNo">1447</span>        final Writes writes = new Writes();<a name="line.1447"></a>
<span class="sourceLineNo">1448</span>        writes.getPuts().put(CURRENT_TERM_KEY, LongEncoder.encode(newTerm));<a name="line.1448"></a>
<span class="sourceLineNo">1449</span>        writes.setRemoves(new KeyRanges(VOTED_FOR_KEY));<a name="line.1449"></a>
<span class="sourceLineNo">1450</span>        try {<a name="line.1450"></a>
<span class="sourceLineNo">1451</span>            this.kv.mutate(writes, true);<a name="line.1451"></a>
<span class="sourceLineNo">1452</span>        } catch (Exception e) {<a name="line.1452"></a>
<span class="sourceLineNo">1453</span>            this.error("error persisting new term " + newTerm, e);<a name="line.1453"></a>
<span class="sourceLineNo">1454</span>            return false;<a name="line.1454"></a>
<span class="sourceLineNo">1455</span>        }<a name="line.1455"></a>
<span class="sourceLineNo">1456</span><a name="line.1456"></a>
<span class="sourceLineNo">1457</span>        // Update in-memory copy<a name="line.1457"></a>
<span class="sourceLineNo">1458</span>        this.currentTerm = newTerm;<a name="line.1458"></a>
<span class="sourceLineNo">1459</span>        return true;<a name="line.1459"></a>
<span class="sourceLineNo">1460</span>    }<a name="line.1460"></a>
<span class="sourceLineNo">1461</span><a name="line.1461"></a>
<span class="sourceLineNo">1462</span>    /**<a name="line.1462"></a>
<span class="sourceLineNo">1463</span>     * Join the specified cluster.<a name="line.1463"></a>
<span class="sourceLineNo">1464</span>     *<a name="line.1464"></a>
<span class="sourceLineNo">1465</span>     * @param newClusterId cluster ID, or zero to have one randomly assigned<a name="line.1465"></a>
<span class="sourceLineNo">1466</span>     */<a name="line.1466"></a>
<span class="sourceLineNo">1467</span>    private boolean joinCluster(int newClusterId) {<a name="line.1467"></a>
<span class="sourceLineNo">1468</span><a name="line.1468"></a>
<span class="sourceLineNo">1469</span>        // Sanity check<a name="line.1469"></a>
<span class="sourceLineNo">1470</span>        assert Thread.holdsLock(this);<a name="line.1470"></a>
<span class="sourceLineNo">1471</span>        Preconditions.checkState(this.clusterId == 0);<a name="line.1471"></a>
<span class="sourceLineNo">1472</span><a name="line.1472"></a>
<span class="sourceLineNo">1473</span>        // Pick a new, random cluster ID if needed<a name="line.1473"></a>
<span class="sourceLineNo">1474</span>        while (newClusterId == 0)<a name="line.1474"></a>
<span class="sourceLineNo">1475</span>            newClusterId = this.random.nextInt();<a name="line.1475"></a>
<span class="sourceLineNo">1476</span><a name="line.1476"></a>
<span class="sourceLineNo">1477</span>        // Persist it<a name="line.1477"></a>
<span class="sourceLineNo">1478</span>        this.info("joining cluster with ID " + String.format("0x%08x", newClusterId));<a name="line.1478"></a>
<span class="sourceLineNo">1479</span>        final Writes writes = new Writes();<a name="line.1479"></a>
<span class="sourceLineNo">1480</span>        writes.getPuts().put(CLUSTER_ID_KEY, LongEncoder.encode(newClusterId));<a name="line.1480"></a>
<span class="sourceLineNo">1481</span>        try {<a name="line.1481"></a>
<span class="sourceLineNo">1482</span>            this.kv.mutate(writes, true);<a name="line.1482"></a>
<span class="sourceLineNo">1483</span>        } catch (Exception e) {<a name="line.1483"></a>
<span class="sourceLineNo">1484</span>            this.error("error updating key/value store with new cluster ID", e);<a name="line.1484"></a>
<span class="sourceLineNo">1485</span>            return false;<a name="line.1485"></a>
<span class="sourceLineNo">1486</span>        }<a name="line.1486"></a>
<span class="sourceLineNo">1487</span><a name="line.1487"></a>
<span class="sourceLineNo">1488</span>        // Done<a name="line.1488"></a>
<span class="sourceLineNo">1489</span>        this.clusterId = newClusterId;<a name="line.1489"></a>
<span class="sourceLineNo">1490</span>        return true;<a name="line.1490"></a>
<span class="sourceLineNo">1491</span>    }<a name="line.1491"></a>
<span class="sourceLineNo">1492</span><a name="line.1492"></a>
<span class="sourceLineNo">1493</span>    /**<a name="line.1493"></a>
<span class="sourceLineNo">1494</span>     * Set the Raft role.<a name="line.1494"></a>
<span class="sourceLineNo">1495</span>     *<a name="line.1495"></a>
<span class="sourceLineNo">1496</span>     * @param role new role<a name="line.1496"></a>
<span class="sourceLineNo">1497</span>     */<a name="line.1497"></a>
<span class="sourceLineNo">1498</span>    private void changeRole(Role role) {<a name="line.1498"></a>
<span class="sourceLineNo">1499</span><a name="line.1499"></a>
<span class="sourceLineNo">1500</span>        // Sanity check<a name="line.1500"></a>
<span class="sourceLineNo">1501</span>        assert Thread.holdsLock(this);<a name="line.1501"></a>
<span class="sourceLineNo">1502</span>        assert role != null;<a name="line.1502"></a>
<span class="sourceLineNo">1503</span><a name="line.1503"></a>
<span class="sourceLineNo">1504</span>        // Shutdown previous role (if any)<a name="line.1504"></a>
<span class="sourceLineNo">1505</span>        if (this.role != null) {<a name="line.1505"></a>
<span class="sourceLineNo">1506</span>            this.role.shutdown();<a name="line.1506"></a>
<span class="sourceLineNo">1507</span>            for (Iterator&lt;Service&gt; i = this.pendingService.iterator(); i.hasNext(); ) {<a name="line.1507"></a>
<span class="sourceLineNo">1508</span>                final Service service = i.next();<a name="line.1508"></a>
<span class="sourceLineNo">1509</span>                if (service.getRole() != null)<a name="line.1509"></a>
<span class="sourceLineNo">1510</span>                    i.remove();<a name="line.1510"></a>
<span class="sourceLineNo">1511</span>            }<a name="line.1511"></a>
<span class="sourceLineNo">1512</span>        }<a name="line.1512"></a>
<span class="sourceLineNo">1513</span><a name="line.1513"></a>
<span class="sourceLineNo">1514</span>        // Setup new role<a name="line.1514"></a>
<span class="sourceLineNo">1515</span>        this.role = role;<a name="line.1515"></a>
<span class="sourceLineNo">1516</span>        this.role.setup();<a name="line.1516"></a>
<span class="sourceLineNo">1517</span>        this.info("changing role to " + role);<a name="line.1517"></a>
<span class="sourceLineNo">1518</span>    }<a name="line.1518"></a>
<span class="sourceLineNo">1519</span><a name="line.1519"></a>
<span class="sourceLineNo">1520</span>    /**<a name="line.1520"></a>
<span class="sourceLineNo">1521</span>     * Append a log entry to the Raft log.<a name="line.1521"></a>
<span class="sourceLineNo">1522</span>     *<a name="line.1522"></a>
<span class="sourceLineNo">1523</span>     * @param term new log entry term<a name="line.1523"></a>
<span class="sourceLineNo">1524</span>     * @param entry entry to add; the {@linkplain NewLogEntry#getTempFile temporary file} must be already durably persisted,<a name="line.1524"></a>
<span class="sourceLineNo">1525</span>     *  and will be renamed<a name="line.1525"></a>
<span class="sourceLineNo">1526</span>     * @return new {@link LogEntry}<a name="line.1526"></a>
<span class="sourceLineNo">1527</span>     * @throws Exception if an error occurs<a name="line.1527"></a>
<span class="sourceLineNo">1528</span>     */<a name="line.1528"></a>
<span class="sourceLineNo">1529</span>    private LogEntry appendLogEntry(long term, NewLogEntry newLogEntry) throws Exception {<a name="line.1529"></a>
<span class="sourceLineNo">1530</span><a name="line.1530"></a>
<span class="sourceLineNo">1531</span>        // Sanity check<a name="line.1531"></a>
<span class="sourceLineNo">1532</span>        assert Thread.holdsLock(this);<a name="line.1532"></a>
<span class="sourceLineNo">1533</span>        assert this.role != null;<a name="line.1533"></a>
<span class="sourceLineNo">1534</span>        assert newLogEntry != null;<a name="line.1534"></a>
<span class="sourceLineNo">1535</span><a name="line.1535"></a>
<span class="sourceLineNo">1536</span>        // Get file length<a name="line.1536"></a>
<span class="sourceLineNo">1537</span>        final LogEntry.Data data = newLogEntry.getData();<a name="line.1537"></a>
<span class="sourceLineNo">1538</span>        final File tempFile = newLogEntry.getTempFile();<a name="line.1538"></a>
<span class="sourceLineNo">1539</span>        final long fileLength = Util.getLength(tempFile);<a name="line.1539"></a>
<span class="sourceLineNo">1540</span><a name="line.1540"></a>
<span class="sourceLineNo">1541</span>        // Create new log entry<a name="line.1541"></a>
<span class="sourceLineNo">1542</span>        final LogEntry logEntry = new LogEntry(term, this.getLastLogIndex() + 1, this.logDir, data, fileLength);<a name="line.1542"></a>
<span class="sourceLineNo">1543</span>        if (this.log.isDebugEnabled())<a name="line.1543"></a>
<span class="sourceLineNo">1544</span>            this.debug("adding new log entry " + logEntry + " using " + tempFile.getName());<a name="line.1544"></a>
<span class="sourceLineNo">1545</span><a name="line.1545"></a>
<span class="sourceLineNo">1546</span>        // Atomically rename file and fsync() directory to durably persist<a name="line.1546"></a>
<span class="sourceLineNo">1547</span>        Files.move(tempFile.toPath(), logEntry.getFile().toPath(), StandardCopyOption.ATOMIC_MOVE);<a name="line.1547"></a>
<span class="sourceLineNo">1548</span>        this.logDirChannel.force(true);<a name="line.1548"></a>
<span class="sourceLineNo">1549</span><a name="line.1549"></a>
<span class="sourceLineNo">1550</span>        // Add new log entry to in-memory log<a name="line.1550"></a>
<span class="sourceLineNo">1551</span>        this.raftLog.add(logEntry);<a name="line.1551"></a>
<span class="sourceLineNo">1552</span><a name="line.1552"></a>
<span class="sourceLineNo">1553</span>        // Update current config<a name="line.1553"></a>
<span class="sourceLineNo">1554</span>        if (logEntry.applyConfigChange(this.currentConfig))<a name="line.1554"></a>
<span class="sourceLineNo">1555</span>            this.info("new current configuration: " + this.currentConfig);<a name="line.1555"></a>
<span class="sourceLineNo">1556</span><a name="line.1556"></a>
<span class="sourceLineNo">1557</span>        // Done<a name="line.1557"></a>
<span class="sourceLineNo">1558</span>        return logEntry;<a name="line.1558"></a>
<span class="sourceLineNo">1559</span>    }<a name="line.1559"></a>
<span class="sourceLineNo">1560</span><a name="line.1560"></a>
<span class="sourceLineNo">1561</span>    private long getLastLogIndex() {<a name="line.1561"></a>
<span class="sourceLineNo">1562</span>        assert Thread.holdsLock(this);<a name="line.1562"></a>
<span class="sourceLineNo">1563</span>        return this.lastAppliedIndex + this.raftLog.size();<a name="line.1563"></a>
<span class="sourceLineNo">1564</span>    }<a name="line.1564"></a>
<span class="sourceLineNo">1565</span><a name="line.1565"></a>
<span class="sourceLineNo">1566</span>    private long getLastLogTerm() {<a name="line.1566"></a>
<span class="sourceLineNo">1567</span>        assert Thread.holdsLock(this);<a name="line.1567"></a>
<span class="sourceLineNo">1568</span>        return this.getLogTermAtIndex(this.getLastLogIndex());<a name="line.1568"></a>
<span class="sourceLineNo">1569</span>    }<a name="line.1569"></a>
<span class="sourceLineNo">1570</span><a name="line.1570"></a>
<span class="sourceLineNo">1571</span>    private long getLogTermAtIndex(long index) {<a name="line.1571"></a>
<span class="sourceLineNo">1572</span>        assert Thread.holdsLock(this);<a name="line.1572"></a>
<span class="sourceLineNo">1573</span>        assert index &gt;= this.lastAppliedIndex;<a name="line.1573"></a>
<span class="sourceLineNo">1574</span>        assert index &lt;= this.getLastLogIndex();<a name="line.1574"></a>
<span class="sourceLineNo">1575</span>        return index == this.lastAppliedIndex ? this.lastAppliedTerm : this.getLogEntryAtIndex(index).getTerm();<a name="line.1575"></a>
<span class="sourceLineNo">1576</span>    }<a name="line.1576"></a>
<span class="sourceLineNo">1577</span><a name="line.1577"></a>
<span class="sourceLineNo">1578</span>    private LogEntry getLogEntryAtIndex(long index) {<a name="line.1578"></a>
<span class="sourceLineNo">1579</span>        assert Thread.holdsLock(this);<a name="line.1579"></a>
<span class="sourceLineNo">1580</span>        assert index &gt; this.lastAppliedIndex;<a name="line.1580"></a>
<span class="sourceLineNo">1581</span>        assert index &lt;= this.getLastLogIndex();<a name="line.1581"></a>
<span class="sourceLineNo">1582</span>        return this.raftLog.get((int)(index - this.lastAppliedIndex - 1));<a name="line.1582"></a>
<span class="sourceLineNo">1583</span>    }<a name="line.1583"></a>
<span class="sourceLineNo">1584</span><a name="line.1584"></a>
<span class="sourceLineNo">1585</span>    /**<a name="line.1585"></a>
<span class="sourceLineNo">1586</span>     * Contains the information required to commit a new entry to the log.<a name="line.1586"></a>
<span class="sourceLineNo">1587</span>     */<a name="line.1587"></a>
<span class="sourceLineNo">1588</span>    private class NewLogEntry {<a name="line.1588"></a>
<span class="sourceLineNo">1589</span><a name="line.1589"></a>
<span class="sourceLineNo">1590</span>        private final LogEntry.Data data;<a name="line.1590"></a>
<span class="sourceLineNo">1591</span>        private final File tempFile;<a name="line.1591"></a>
<span class="sourceLineNo">1592</span><a name="line.1592"></a>
<span class="sourceLineNo">1593</span>        /**<a name="line.1593"></a>
<span class="sourceLineNo">1594</span>         * Create an instance from a transaction and a temporary file<a name="line.1594"></a>
<span class="sourceLineNo">1595</span>         *<a name="line.1595"></a>
<span class="sourceLineNo">1596</span>         * @param data log entry mutations<a name="line.1596"></a>
<span class="sourceLineNo">1597</span>         * @throws Exception if an error occurs<a name="line.1597"></a>
<span class="sourceLineNo">1598</span>         */<a name="line.1598"></a>
<span class="sourceLineNo">1599</span>        public NewLogEntry(RaftKVTransaction tx, File tempFile) throws IOException {<a name="line.1599"></a>
<span class="sourceLineNo">1600</span>            this.data = new LogEntry.Data(tx.getMutableView().getWrites(), tx.getConfigChange());<a name="line.1600"></a>
<span class="sourceLineNo">1601</span>            this.tempFile = tempFile;<a name="line.1601"></a>
<span class="sourceLineNo">1602</span>        }<a name="line.1602"></a>
<span class="sourceLineNo">1603</span><a name="line.1603"></a>
<span class="sourceLineNo">1604</span>        /**<a name="line.1604"></a>
<span class="sourceLineNo">1605</span>         * Create an instance from a transaction.<a name="line.1605"></a>
<span class="sourceLineNo">1606</span>         *<a name="line.1606"></a>
<span class="sourceLineNo">1607</span>         * @param data log entry mutations<a name="line.1607"></a>
<span class="sourceLineNo">1608</span>         * @throws Exception if an error occurs<a name="line.1608"></a>
<span class="sourceLineNo">1609</span>         */<a name="line.1609"></a>
<span class="sourceLineNo">1610</span>        public NewLogEntry(RaftKVTransaction tx) throws IOException {<a name="line.1610"></a>
<span class="sourceLineNo">1611</span>            this.data = new LogEntry.Data(tx.getMutableView().getWrites(), tx.getConfigChange());<a name="line.1611"></a>
<span class="sourceLineNo">1612</span>            this.tempFile = File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX, RaftKVDatabase.this.logDir);<a name="line.1612"></a>
<span class="sourceLineNo">1613</span>            try (FileWriter output = new FileWriter(this.tempFile)) {<a name="line.1613"></a>
<span class="sourceLineNo">1614</span>                LogEntry.writeData(output, data);<a name="line.1614"></a>
<span class="sourceLineNo">1615</span>            }<a name="line.1615"></a>
<span class="sourceLineNo">1616</span>        }<a name="line.1616"></a>
<span class="sourceLineNo">1617</span><a name="line.1617"></a>
<span class="sourceLineNo">1618</span>        /**<a name="line.1618"></a>
<span class="sourceLineNo">1619</span>         * Create an instance from a {@link LogEntry.Data} object.<a name="line.1619"></a>
<span class="sourceLineNo">1620</span>         *<a name="line.1620"></a>
<span class="sourceLineNo">1621</span>         * @param data mutation data<a name="line.1621"></a>
<span class="sourceLineNo">1622</span>         * @throws Exception if an error occurs<a name="line.1622"></a>
<span class="sourceLineNo">1623</span>         */<a name="line.1623"></a>
<span class="sourceLineNo">1624</span>        public NewLogEntry(LogEntry.Data data) throws IOException {<a name="line.1624"></a>
<span class="sourceLineNo">1625</span>            this.data = data;<a name="line.1625"></a>
<span class="sourceLineNo">1626</span>            this.tempFile = File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX, RaftKVDatabase.this.logDir);<a name="line.1626"></a>
<span class="sourceLineNo">1627</span>            try (FileWriter output = new FileWriter(this.tempFile)) {<a name="line.1627"></a>
<span class="sourceLineNo">1628</span>                LogEntry.writeData(output, data);<a name="line.1628"></a>
<span class="sourceLineNo">1629</span>            }<a name="line.1629"></a>
<span class="sourceLineNo">1630</span>        }<a name="line.1630"></a>
<span class="sourceLineNo">1631</span><a name="line.1631"></a>
<span class="sourceLineNo">1632</span>        /**<a name="line.1632"></a>
<span class="sourceLineNo">1633</span>         * Create an instance from a serialized data in a {@link ByteBuffer}.<a name="line.1633"></a>
<span class="sourceLineNo">1634</span>         *<a name="line.1634"></a>
<span class="sourceLineNo">1635</span>         * @param buf buffer containing serialized mutations<a name="line.1635"></a>
<span class="sourceLineNo">1636</span>         * @throws Exception if an error occurs<a name="line.1636"></a>
<span class="sourceLineNo">1637</span>         */<a name="line.1637"></a>
<span class="sourceLineNo">1638</span>        public NewLogEntry(ByteBuffer dataBuf) throws IOException {<a name="line.1638"></a>
<span class="sourceLineNo">1639</span><a name="line.1639"></a>
<span class="sourceLineNo">1640</span>            // Copy data to temporary file<a name="line.1640"></a>
<span class="sourceLineNo">1641</span>            this.tempFile = File.createTempFile(TEMP_FILE_PREFIX, TEMP_FILE_SUFFIX, RaftKVDatabase.this.logDir);<a name="line.1641"></a>
<span class="sourceLineNo">1642</span>            try (FileWriter output = new FileWriter(this.tempFile)) {<a name="line.1642"></a>
<span class="sourceLineNo">1643</span>                while (dataBuf.hasRemaining())<a name="line.1643"></a>
<span class="sourceLineNo">1644</span>                    output.getFileOutputStream().getChannel().write(dataBuf);<a name="line.1644"></a>
<span class="sourceLineNo">1645</span>            }<a name="line.1645"></a>
<span class="sourceLineNo">1646</span><a name="line.1646"></a>
<span class="sourceLineNo">1647</span>            // Avoid having two copies of the data in memory at once<a name="line.1647"></a>
<span class="sourceLineNo">1648</span>            dataBuf = null;<a name="line.1648"></a>
<span class="sourceLineNo">1649</span><a name="line.1649"></a>
<span class="sourceLineNo">1650</span>            // Deserialize data from file back into memory<a name="line.1650"></a>
<span class="sourceLineNo">1651</span>            try (BufferedInputStream input = new BufferedInputStream(new FileInputStream(tempFile), 4096)) {<a name="line.1651"></a>
<span class="sourceLineNo">1652</span>                this.data = LogEntry.readData(input);<a name="line.1652"></a>
<span class="sourceLineNo">1653</span>            }<a name="line.1653"></a>
<span class="sourceLineNo">1654</span>        }<a name="line.1654"></a>
<span class="sourceLineNo">1655</span><a name="line.1655"></a>
<span class="sourceLineNo">1656</span>        public LogEntry.Data getData() {<a name="line.1656"></a>
<span class="sourceLineNo">1657</span>            return this.data;<a name="line.1657"></a>
<span class="sourceLineNo">1658</span>        }<a name="line.1658"></a>
<span class="sourceLineNo">1659</span><a name="line.1659"></a>
<span class="sourceLineNo">1660</span>        public File getTempFile() {<a name="line.1660"></a>
<span class="sourceLineNo">1661</span>            return this.tempFile;<a name="line.1661"></a>
<span class="sourceLineNo">1662</span>        }<a name="line.1662"></a>
<span class="sourceLineNo">1663</span><a name="line.1663"></a>
<span class="sourceLineNo">1664</span>        public void cancel() {<a name="line.1664"></a>
<span class="sourceLineNo">1665</span>            this.tempFile.delete();<a name="line.1665"></a>
<span class="sourceLineNo">1666</span>        }<a name="line.1666"></a>
<span class="sourceLineNo">1667</span>    }<a name="line.1667"></a>
<span class="sourceLineNo">1668</span><a name="line.1668"></a>
<span class="sourceLineNo">1669</span>// Object<a name="line.1669"></a>
<span class="sourceLineNo">1670</span><a name="line.1670"></a>
<span class="sourceLineNo">1671</span>    @Override<a name="line.1671"></a>
<span class="sourceLineNo">1672</span>    public String toString() {<a name="line.1672"></a>
<span class="sourceLineNo">1673</span>        return this.getClass().getSimpleName()<a name="line.1673"></a>
<span class="sourceLineNo">1674</span>          + "[identity=" + (this.identity != null ? "\"" + this.identity + "\"" : null)<a name="line.1674"></a>
<span class="sourceLineNo">1675</span>          + ",logDir=" + this.logDir<a name="line.1675"></a>
<span class="sourceLineNo">1676</span>          + ",term=" + this.currentTerm<a name="line.1676"></a>
<span class="sourceLineNo">1677</span>          + ",commitIndex=" + this.commitIndex<a name="line.1677"></a>
<span class="sourceLineNo">1678</span>          + ",lastApplied=" + this.lastAppliedTerm + "/" + this.lastAppliedIndex<a name="line.1678"></a>
<span class="sourceLineNo">1679</span>          + ",raftLog=" + this.raftLog<a name="line.1679"></a>
<span class="sourceLineNo">1680</span>          + ",role=" + this.role<a name="line.1680"></a>
<span class="sourceLineNo">1681</span>          + (this.shuttingDown ? ",shuttingDown" : "")<a name="line.1681"></a>
<span class="sourceLineNo">1682</span>          + "]";<a name="line.1682"></a>
<span class="sourceLineNo">1683</span>    }<a name="line.1683"></a>
<span class="sourceLineNo">1684</span><a name="line.1684"></a>
<span class="sourceLineNo">1685</span>// Network.Handler and Messaging<a name="line.1685"></a>
<span class="sourceLineNo">1686</span><a name="line.1686"></a>
<span class="sourceLineNo">1687</span>    private void handle(String sender, ByteBuffer buf) {<a name="line.1687"></a>
<span class="sourceLineNo">1688</span><a name="line.1688"></a>
<span class="sourceLineNo">1689</span>        // Decode message<a name="line.1689"></a>
<span class="sourceLineNo">1690</span>        final Message msg;<a name="line.1690"></a>
<span class="sourceLineNo">1691</span>        try {<a name="line.1691"></a>
<span class="sourceLineNo">1692</span>            msg = Message.decode(buf);<a name="line.1692"></a>
<span class="sourceLineNo">1693</span>        } catch (IllegalArgumentException e) {<a name="line.1693"></a>
<span class="sourceLineNo">1694</span>            this.error("rec'd bogus message from " + sender + ", ignoring", e);<a name="line.1694"></a>
<span class="sourceLineNo">1695</span>            return;<a name="line.1695"></a>
<span class="sourceLineNo">1696</span>        }<a name="line.1696"></a>
<span class="sourceLineNo">1697</span><a name="line.1697"></a>
<span class="sourceLineNo">1698</span>        // Receive message<a name="line.1698"></a>
<span class="sourceLineNo">1699</span>        this.receiveMessage(sender, msg);<a name="line.1699"></a>
<span class="sourceLineNo">1700</span>    }<a name="line.1700"></a>
<span class="sourceLineNo">1701</span><a name="line.1701"></a>
<span class="sourceLineNo">1702</span>    private synchronized void outputQueueEmpty(String address) {<a name="line.1702"></a>
<span class="sourceLineNo">1703</span><a name="line.1703"></a>
<span class="sourceLineNo">1704</span>        // Sanity check<a name="line.1704"></a>
<span class="sourceLineNo">1705</span>        assert this.checkState();<a name="line.1705"></a>
<span class="sourceLineNo">1706</span><a name="line.1706"></a>
<span class="sourceLineNo">1707</span>        // Update transmitting status<a name="line.1707"></a>
<span class="sourceLineNo">1708</span>        if (!this.transmitting.remove(address))<a name="line.1708"></a>
<span class="sourceLineNo">1709</span>            return;<a name="line.1709"></a>
<span class="sourceLineNo">1710</span>        if (this.log.isTraceEnabled())<a name="line.1710"></a>
<span class="sourceLineNo">1711</span>            this.trace("QUEUE_EMPTY address " + address + " in " + this.role);<a name="line.1711"></a>
<span class="sourceLineNo">1712</span><a name="line.1712"></a>
<span class="sourceLineNo">1713</span>        // Notify role<a name="line.1713"></a>
<span class="sourceLineNo">1714</span>        if (this.role == null)<a name="line.1714"></a>
<span class="sourceLineNo">1715</span>            return;<a name="line.1715"></a>
<span class="sourceLineNo">1716</span>        this.role.outputQueueEmpty(address);<a name="line.1716"></a>
<span class="sourceLineNo">1717</span>    }<a name="line.1717"></a>
<span class="sourceLineNo">1718</span><a name="line.1718"></a>
<span class="sourceLineNo">1719</span>    private boolean isTransmitting(String address) {<a name="line.1719"></a>
<span class="sourceLineNo">1720</span>        return this.transmitting.contains(address);<a name="line.1720"></a>
<span class="sourceLineNo">1721</span>    }<a name="line.1721"></a>
<span class="sourceLineNo">1722</span><a name="line.1722"></a>
<span class="sourceLineNo">1723</span>// Messages<a name="line.1723"></a>
<span class="sourceLineNo">1724</span><a name="line.1724"></a>
<span class="sourceLineNo">1725</span>    private boolean sendMessage(Message msg) {<a name="line.1725"></a>
<span class="sourceLineNo">1726</span><a name="line.1726"></a>
<span class="sourceLineNo">1727</span>        // Sanity check<a name="line.1727"></a>
<span class="sourceLineNo">1728</span>        assert Thread.holdsLock(this);<a name="line.1728"></a>
<span class="sourceLineNo">1729</span><a name="line.1729"></a>
<span class="sourceLineNo">1730</span>        // Get peer's address; if unknown, use the return address of the message being processed (if any)<a name="line.1730"></a>
<span class="sourceLineNo">1731</span>        final String peer = msg.getRecipientId();<a name="line.1731"></a>
<span class="sourceLineNo">1732</span>        String address = this.currentConfig.get(peer);<a name="line.1732"></a>
<span class="sourceLineNo">1733</span>        if (address == null)<a name="line.1733"></a>
<span class="sourceLineNo">1734</span>            address = this.returnAddress;<a name="line.1734"></a>
<span class="sourceLineNo">1735</span>        if (address == null) {<a name="line.1735"></a>
<span class="sourceLineNo">1736</span>            this.warn("can't send " + msg + " to unknown peer \"" + peer + "\"");<a name="line.1736"></a>
<span class="sourceLineNo">1737</span>            return false;<a name="line.1737"></a>
<span class="sourceLineNo">1738</span>        }<a name="line.1738"></a>
<span class="sourceLineNo">1739</span><a name="line.1739"></a>
<span class="sourceLineNo">1740</span>        // Send message<a name="line.1740"></a>
<span class="sourceLineNo">1741</span>        if (this.log.isTraceEnabled())<a name="line.1741"></a>
<span class="sourceLineNo">1742</span>            this.trace("XMIT " + msg + " to " + address);<a name="line.1742"></a>
<span class="sourceLineNo">1743</span>        if (this.network.send(address, msg.encode())) {<a name="line.1743"></a>
<span class="sourceLineNo">1744</span>            this.transmitting.add(address);<a name="line.1744"></a>
<span class="sourceLineNo">1745</span>            return true;<a name="line.1745"></a>
<span class="sourceLineNo">1746</span>        }<a name="line.1746"></a>
<span class="sourceLineNo">1747</span><a name="line.1747"></a>
<span class="sourceLineNo">1748</span>        // Transmit failed<a name="line.1748"></a>
<span class="sourceLineNo">1749</span>        this.warn("transmit of " + msg + " to \"" + peer + "\" failed locally");<a name="line.1749"></a>
<span class="sourceLineNo">1750</span>        return false;<a name="line.1750"></a>
<span class="sourceLineNo">1751</span>    }<a name="line.1751"></a>
<span class="sourceLineNo">1752</span><a name="line.1752"></a>
<span class="sourceLineNo">1753</span>    private synchronized void receiveMessage(String address, Message msg) {<a name="line.1753"></a>
<span class="sourceLineNo">1754</span><a name="line.1754"></a>
<span class="sourceLineNo">1755</span>        // Sanity check<a name="line.1755"></a>
<span class="sourceLineNo">1756</span>        assert Thread.holdsLock(this);<a name="line.1756"></a>
<span class="sourceLineNo">1757</span>        assert this.checkState();<a name="line.1757"></a>
<span class="sourceLineNo">1758</span>        if (this.role == null) {<a name="line.1758"></a>
<span class="sourceLineNo">1759</span>            if (this.log.isDebugEnabled())<a name="line.1759"></a>
<span class="sourceLineNo">1760</span>                this.debug("rec'd " + msg + " rec'd in shutdown state; ignoring");<a name="line.1760"></a>
<span class="sourceLineNo">1761</span>            return;<a name="line.1761"></a>
<span class="sourceLineNo">1762</span>        }<a name="line.1762"></a>
<span class="sourceLineNo">1763</span><a name="line.1763"></a>
<span class="sourceLineNo">1764</span>        // Sanity check cluster ID<a name="line.1764"></a>
<span class="sourceLineNo">1765</span>        if (msg.getClusterId() == 0) {<a name="line.1765"></a>
<span class="sourceLineNo">1766</span>            this.warn("rec'd " + msg + " with zero cluster ID from " + address + "; ignoring");<a name="line.1766"></a>
<span class="sourceLineNo">1767</span>            return;<a name="line.1767"></a>
<span class="sourceLineNo">1768</span>        }<a name="line.1768"></a>
<span class="sourceLineNo">1769</span>        if (this.clusterId != 0 &amp;&amp; msg.getClusterId() != this.clusterId) {<a name="line.1769"></a>
<span class="sourceLineNo">1770</span>            this.warn("rec'd " + msg + " with foreign cluster ID "<a name="line.1770"></a>
<span class="sourceLineNo">1771</span>              + String.format("0x%08x", msg.getClusterId()) + " != " + String.format("0x%08x", this.clusterId) + "; ignoring");<a name="line.1771"></a>
<span class="sourceLineNo">1772</span>            return;<a name="line.1772"></a>
<span class="sourceLineNo">1773</span>        }<a name="line.1773"></a>
<span class="sourceLineNo">1774</span><a name="line.1774"></a>
<span class="sourceLineNo">1775</span>        // Sanity check sender<a name="line.1775"></a>
<span class="sourceLineNo">1776</span>        final String peer = msg.getSenderId();<a name="line.1776"></a>
<span class="sourceLineNo">1777</span>        if (peer.equals(this.identity)) {<a name="line.1777"></a>
<span class="sourceLineNo">1778</span>            this.warn("rec'd " + msg + " from myself (\"" + peer + "\", address " + address + "); ignoring");<a name="line.1778"></a>
<span class="sourceLineNo">1779</span>            return;<a name="line.1779"></a>
<span class="sourceLineNo">1780</span>        }<a name="line.1780"></a>
<span class="sourceLineNo">1781</span><a name="line.1781"></a>
<span class="sourceLineNo">1782</span>        // Sanity check recipient<a name="line.1782"></a>
<span class="sourceLineNo">1783</span>        final String dest = msg.getRecipientId();<a name="line.1783"></a>
<span class="sourceLineNo">1784</span>        if (!dest.equals(this.identity)) {<a name="line.1784"></a>
<span class="sourceLineNo">1785</span>            this.warn("rec'd misdirected " + msg + " intended for \"" + dest + "\" from " + address + "; ignoring");<a name="line.1785"></a>
<span class="sourceLineNo">1786</span>            return;<a name="line.1786"></a>
<span class="sourceLineNo">1787</span>        }<a name="line.1787"></a>
<span class="sourceLineNo">1788</span><a name="line.1788"></a>
<span class="sourceLineNo">1789</span>        // Is sender's term too low? Ignore it<a name="line.1789"></a>
<span class="sourceLineNo">1790</span>        if (msg.getTerm() &lt; this.currentTerm) {<a name="line.1790"></a>
<span class="sourceLineNo">1791</span>            this.info("rec'd " + msg + " with term " + msg.getTerm() + " &lt; " + this.currentTerm + " from \""<a name="line.1791"></a>
<span class="sourceLineNo">1792</span>              + peer + "\" at " + address + ", ignoring");<a name="line.1792"></a>
<span class="sourceLineNo">1793</span>            return;<a name="line.1793"></a>
<span class="sourceLineNo">1794</span>        }<a name="line.1794"></a>
<span class="sourceLineNo">1795</span><a name="line.1795"></a>
<span class="sourceLineNo">1796</span>        // Is my term too low? If so update and revert to follower<a name="line.1796"></a>
<span class="sourceLineNo">1797</span>        if (msg.getTerm() &gt; this.currentTerm) {<a name="line.1797"></a>
<span class="sourceLineNo">1798</span><a name="line.1798"></a>
<span class="sourceLineNo">1799</span>            // First check with current role; in some special cases we ignore this<a name="line.1799"></a>
<span class="sourceLineNo">1800</span>            if (!this.role.mayAdvanceCurrentTerm(msg)) {<a name="line.1800"></a>
<span class="sourceLineNo">1801</span>                if (this.log.isTraceEnabled()) {<a name="line.1801"></a>
<span class="sourceLineNo">1802</span>                    this.trace("rec'd " + msg + " with term " + msg.getTerm() + " &gt; " + this.currentTerm + " from \""<a name="line.1802"></a>
<span class="sourceLineNo">1803</span>                      + peer + "\" but current role says to ignore it");<a name="line.1803"></a>
<span class="sourceLineNo">1804</span>                }<a name="line.1804"></a>
<span class="sourceLineNo">1805</span>                return;<a name="line.1805"></a>
<span class="sourceLineNo">1806</span>            }<a name="line.1806"></a>
<span class="sourceLineNo">1807</span><a name="line.1807"></a>
<span class="sourceLineNo">1808</span>            // Revert to follower<a name="line.1808"></a>
<span class="sourceLineNo">1809</span>            this.info("rec'd " + msg.getClass().getSimpleName() + " with term " + msg.getTerm() + " &gt; " + this.currentTerm<a name="line.1809"></a>
<span class="sourceLineNo">1810</span>              + " from \"" + peer + "\", updating term and "<a name="line.1810"></a>
<span class="sourceLineNo">1811</span>              + (this.role instanceof FollowerRole ? "remaining a" : "reverting to") + " follower");<a name="line.1811"></a>
<span class="sourceLineNo">1812</span>            if (!this.advanceTerm(msg.getTerm()))<a name="line.1812"></a>
<span class="sourceLineNo">1813</span>                return;<a name="line.1813"></a>
<span class="sourceLineNo">1814</span>            this.changeRole(msg.isLeaderMessage() ? new FollowerRole(this, peer, address) : new FollowerRole(this));<a name="line.1814"></a>
<span class="sourceLineNo">1815</span>        }<a name="line.1815"></a>
<span class="sourceLineNo">1816</span><a name="line.1816"></a>
<span class="sourceLineNo">1817</span>        // Debug<a name="line.1817"></a>
<span class="sourceLineNo">1818</span>        if (this.log.isTraceEnabled())<a name="line.1818"></a>
<span class="sourceLineNo">1819</span>            this.trace("RECV " + msg + " in " + this.role + " from " + address);<a name="line.1819"></a>
<span class="sourceLineNo">1820</span><a name="line.1820"></a>
<span class="sourceLineNo">1821</span>        // Handle message<a name="line.1821"></a>
<span class="sourceLineNo">1822</span>        this.returnAddress = address;<a name="line.1822"></a>
<span class="sourceLineNo">1823</span>        try {<a name="line.1823"></a>
<span class="sourceLineNo">1824</span>            msg.visit(this.role);<a name="line.1824"></a>
<span class="sourceLineNo">1825</span>        } finally {<a name="line.1825"></a>
<span class="sourceLineNo">1826</span>            this.returnAddress = null;<a name="line.1826"></a>
<span class="sourceLineNo">1827</span>        }<a name="line.1827"></a>
<span class="sourceLineNo">1828</span>    }<a name="line.1828"></a>
<span class="sourceLineNo">1829</span><a name="line.1829"></a>
<span class="sourceLineNo">1830</span>// Utility methods<a name="line.1830"></a>
<span class="sourceLineNo">1831</span><a name="line.1831"></a>
<span class="sourceLineNo">1832</span>    private long decodeLong(byte[] key, long defaultValue) throws IOException {<a name="line.1832"></a>
<span class="sourceLineNo">1833</span>        final byte[] value = this.kv.get(key);<a name="line.1833"></a>
<span class="sourceLineNo">1834</span>        if (value == null)<a name="line.1834"></a>
<span class="sourceLineNo">1835</span>            return defaultValue;<a name="line.1835"></a>
<span class="sourceLineNo">1836</span>        try {<a name="line.1836"></a>
<span class="sourceLineNo">1837</span>            return LongEncoder.decode(value);<a name="line.1837"></a>
<span class="sourceLineNo">1838</span>        } catch (IllegalArgumentException e) {<a name="line.1838"></a>
<span class="sourceLineNo">1839</span>            throw new IOException("can't interpret encoded long value "<a name="line.1839"></a>
<span class="sourceLineNo">1840</span>              + ByteUtil.toString(value) + " under key " + ByteUtil.toString(key), e);<a name="line.1840"></a>
<span class="sourceLineNo">1841</span>        }<a name="line.1841"></a>
<span class="sourceLineNo">1842</span>    }<a name="line.1842"></a>
<span class="sourceLineNo">1843</span><a name="line.1843"></a>
<span class="sourceLineNo">1844</span>    private String decodeString(byte[] key, String defaultValue) throws IOException {<a name="line.1844"></a>
<span class="sourceLineNo">1845</span>        final byte[] value = this.kv.get(key);<a name="line.1845"></a>
<span class="sourceLineNo">1846</span>        if (value == null)<a name="line.1846"></a>
<span class="sourceLineNo">1847</span>            return defaultValue;<a name="line.1847"></a>
<span class="sourceLineNo">1848</span>        final DataInputStream input = new DataInputStream(new ByteArrayInputStream(value));<a name="line.1848"></a>
<span class="sourceLineNo">1849</span>        try {<a name="line.1849"></a>
<span class="sourceLineNo">1850</span>            return input.readUTF();<a name="line.1850"></a>
<span class="sourceLineNo">1851</span>        } catch (IOException e) {<a name="line.1851"></a>
<span class="sourceLineNo">1852</span>            throw new IOException("can't interpret encoded string value "<a name="line.1852"></a>
<span class="sourceLineNo">1853</span>              + ByteUtil.toString(value) + " under key " + ByteUtil.toString(key), e);<a name="line.1853"></a>
<span class="sourceLineNo">1854</span>        }<a name="line.1854"></a>
<span class="sourceLineNo">1855</span>    }<a name="line.1855"></a>
<span class="sourceLineNo">1856</span><a name="line.1856"></a>
<span class="sourceLineNo">1857</span>    private byte[] encodeString(String value) {<a name="line.1857"></a>
<span class="sourceLineNo">1858</span>        final ByteArrayOutputStream buf = new ByteArrayOutputStream();<a name="line.1858"></a>
<span class="sourceLineNo">1859</span>        final DataOutputStream output = new DataOutputStream(buf);<a name="line.1859"></a>
<span class="sourceLineNo">1860</span>        try {<a name="line.1860"></a>
<span class="sourceLineNo">1861</span>            output.writeUTF(value);<a name="line.1861"></a>
<span class="sourceLineNo">1862</span>            output.flush();<a name="line.1862"></a>
<span class="sourceLineNo">1863</span>        } catch (IOException e) {<a name="line.1863"></a>
<span class="sourceLineNo">1864</span>            throw new RuntimeException("unexpected error", e);<a name="line.1864"></a>
<span class="sourceLineNo">1865</span>        }<a name="line.1865"></a>
<span class="sourceLineNo">1866</span>        return buf.toByteArray();<a name="line.1866"></a>
<span class="sourceLineNo">1867</span>    }<a name="line.1867"></a>
<span class="sourceLineNo">1868</span><a name="line.1868"></a>
<span class="sourceLineNo">1869</span>    private Map&lt;String, String&gt; decodeConfig(byte[] key) throws IOException {<a name="line.1869"></a>
<span class="sourceLineNo">1870</span>        final Map&lt;String, String&gt; config = new HashMap&lt;&gt;();<a name="line.1870"></a>
<span class="sourceLineNo">1871</span>        final byte[] value = this.kv.get(key);<a name="line.1871"></a>
<span class="sourceLineNo">1872</span>        if (value == null)<a name="line.1872"></a>
<span class="sourceLineNo">1873</span>            return config;<a name="line.1873"></a>
<span class="sourceLineNo">1874</span>        try {<a name="line.1874"></a>
<span class="sourceLineNo">1875</span>            final DataInputStream data = new DataInputStream(new ByteArrayInputStream(value));<a name="line.1875"></a>
<span class="sourceLineNo">1876</span>            while (true) {<a name="line.1876"></a>
<span class="sourceLineNo">1877</span>                data.mark(1);<a name="line.1877"></a>
<span class="sourceLineNo">1878</span>                if (data.read() == -1)<a name="line.1878"></a>
<span class="sourceLineNo">1879</span>                    break;<a name="line.1879"></a>
<span class="sourceLineNo">1880</span>                data.reset();<a name="line.1880"></a>
<span class="sourceLineNo">1881</span>                config.put(data.readUTF(), data.readUTF());<a name="line.1881"></a>
<span class="sourceLineNo">1882</span>            }<a name="line.1882"></a>
<span class="sourceLineNo">1883</span>        } catch (IOException e) {<a name="line.1883"></a>
<span class="sourceLineNo">1884</span>            throw new IOException("can't interpret encoded config "<a name="line.1884"></a>
<span class="sourceLineNo">1885</span>              + ByteUtil.toString(value) + " under key " + ByteUtil.toString(key), e);<a name="line.1885"></a>
<span class="sourceLineNo">1886</span>        }<a name="line.1886"></a>
<span class="sourceLineNo">1887</span>        return config;<a name="line.1887"></a>
<span class="sourceLineNo">1888</span>    }<a name="line.1888"></a>
<span class="sourceLineNo">1889</span><a name="line.1889"></a>
<span class="sourceLineNo">1890</span>    private byte[] encodeConfig(Map&lt;String, String&gt; config) {<a name="line.1890"></a>
<span class="sourceLineNo">1891</span>        final ByteArrayOutputStream buf = new ByteArrayOutputStream();<a name="line.1891"></a>
<span class="sourceLineNo">1892</span>        final DataOutputStream data = new DataOutputStream(buf);<a name="line.1892"></a>
<span class="sourceLineNo">1893</span>        try {<a name="line.1893"></a>
<span class="sourceLineNo">1894</span>            for (Map.Entry&lt;String, String&gt; entry : config.entrySet()) {<a name="line.1894"></a>
<span class="sourceLineNo">1895</span>                data.writeUTF(entry.getKey());<a name="line.1895"></a>
<span class="sourceLineNo">1896</span>                data.writeUTF(entry.getValue());<a name="line.1896"></a>
<span class="sourceLineNo">1897</span>            }<a name="line.1897"></a>
<span class="sourceLineNo">1898</span>            data.flush();<a name="line.1898"></a>
<span class="sourceLineNo">1899</span>        } catch (IOException e) {<a name="line.1899"></a>
<span class="sourceLineNo">1900</span>            throw new RuntimeException("unexpected error", e);<a name="line.1900"></a>
<span class="sourceLineNo">1901</span>        }<a name="line.1901"></a>
<span class="sourceLineNo">1902</span>        return buf.toByteArray();<a name="line.1902"></a>
<span class="sourceLineNo">1903</span>    }<a name="line.1903"></a>
<span class="sourceLineNo">1904</span><a name="line.1904"></a>
<span class="sourceLineNo">1905</span>// Logging<a name="line.1905"></a>
<span class="sourceLineNo">1906</span><a name="line.1906"></a>
<span class="sourceLineNo">1907</span>    private void trace(String msg, Throwable t) {<a name="line.1907"></a>
<span class="sourceLineNo">1908</span>        this.log.trace(String.format("%s %s: %s", new Timestamp(), this.identity, msg), t);<a name="line.1908"></a>
<span class="sourceLineNo">1909</span>    }<a name="line.1909"></a>
<span class="sourceLineNo">1910</span><a name="line.1910"></a>
<span class="sourceLineNo">1911</span>    private void trace(String msg) {<a name="line.1911"></a>
<span class="sourceLineNo">1912</span>        this.log.trace(String.format("%s %s: %s", new Timestamp(), this.identity, msg));<a name="line.1912"></a>
<span class="sourceLineNo">1913</span>    }<a name="line.1913"></a>
<span class="sourceLineNo">1914</span><a name="line.1914"></a>
<span class="sourceLineNo">1915</span>    private void debug(String msg, Throwable t) {<a name="line.1915"></a>
<span class="sourceLineNo">1916</span>        this.log.debug(String.format("%s %s: %s", new Timestamp(), this.identity, msg), t);<a name="line.1916"></a>
<span class="sourceLineNo">1917</span>    }<a name="line.1917"></a>
<span class="sourceLineNo">1918</span><a name="line.1918"></a>
<span class="sourceLineNo">1919</span>    private void debug(String msg) {<a name="line.1919"></a>
<span class="sourceLineNo">1920</span>        this.log.debug(String.format("%s %s: %s", new Timestamp(), this.identity, msg));<a name="line.1920"></a>
<span class="sourceLineNo">1921</span>    }<a name="line.1921"></a>
<span class="sourceLineNo">1922</span><a name="line.1922"></a>
<span class="sourceLineNo">1923</span>    private void info(String msg, Throwable t) {<a name="line.1923"></a>
<span class="sourceLineNo">1924</span>        this.log.info(String.format("%s %s: %s", new Timestamp(), this.identity, msg), t);<a name="line.1924"></a>
<span class="sourceLineNo">1925</span>    }<a name="line.1925"></a>
<span class="sourceLineNo">1926</span><a name="line.1926"></a>
<span class="sourceLineNo">1927</span>    private void info(String msg) {<a name="line.1927"></a>
<span class="sourceLineNo">1928</span>        this.log.info(String.format("%s %s: %s", new Timestamp(), this.identity, msg));<a name="line.1928"></a>
<span class="sourceLineNo">1929</span>    }<a name="line.1929"></a>
<span class="sourceLineNo">1930</span><a name="line.1930"></a>
<span class="sourceLineNo">1931</span>    private void warn(String msg, Throwable t) {<a name="line.1931"></a>
<span class="sourceLineNo">1932</span>        this.log.warn(String.format("%s %s: %s", new Timestamp(), this.identity, msg), t);<a name="line.1932"></a>
<span class="sourceLineNo">1933</span>    }<a name="line.1933"></a>
<span class="sourceLineNo">1934</span><a name="line.1934"></a>
<span class="sourceLineNo">1935</span>    private void warn(String msg) {<a name="line.1935"></a>
<span class="sourceLineNo">1936</span>        this.log.warn(String.format("%s %s: %s", new Timestamp(), this.identity, msg));<a name="line.1936"></a>
<span class="sourceLineNo">1937</span>    }<a name="line.1937"></a>
<span class="sourceLineNo">1938</span><a name="line.1938"></a>
<span class="sourceLineNo">1939</span>    private void error(String msg, Throwable t) {<a name="line.1939"></a>
<span class="sourceLineNo">1940</span>        this.log.error(String.format("%s %s: %s", new Timestamp(), this.identity, msg), t);<a name="line.1940"></a>
<span class="sourceLineNo">1941</span>    }<a name="line.1941"></a>
<span class="sourceLineNo">1942</span><a name="line.1942"></a>
<span class="sourceLineNo">1943</span>    private void error(String msg) {<a name="line.1943"></a>
<span class="sourceLineNo">1944</span>        this.log.error(String.format("%s %s: %s", new Timestamp(), this.identity, msg));<a name="line.1944"></a>
<span class="sourceLineNo">1945</span>    }<a name="line.1945"></a>
<span class="sourceLineNo">1946</span><a name="line.1946"></a>
<span class="sourceLineNo">1947</span>// Raft Roles<a name="line.1947"></a>
<span class="sourceLineNo">1948</span><a name="line.1948"></a>
<span class="sourceLineNo">1949</span>    private abstract static class Role implements MessageSwitch {<a name="line.1949"></a>
<span class="sourceLineNo">1950</span><a name="line.1950"></a>
<span class="sourceLineNo">1951</span>        protected final Logger log = LoggerFactory.getLogger(this.getClass());<a name="line.1951"></a>
<span class="sourceLineNo">1952</span>        protected final RaftKVDatabase raft;<a name="line.1952"></a>
<span class="sourceLineNo">1953</span>        protected final Service checkReadyTransactionsService = new Service(this, "check ready transactions") {<a name="line.1953"></a>
<span class="sourceLineNo">1954</span>            @Override<a name="line.1954"></a>
<span class="sourceLineNo">1955</span>            public void run() {<a name="line.1955"></a>
<span class="sourceLineNo">1956</span>                Role.this.checkReadyTransactions();<a name="line.1956"></a>
<span class="sourceLineNo">1957</span>            }<a name="line.1957"></a>
<span class="sourceLineNo">1958</span>        };<a name="line.1958"></a>
<span class="sourceLineNo">1959</span>        protected final Service checkWaitingTransactionsService = new Service(this, "check waiting transactions") {<a name="line.1959"></a>
<span class="sourceLineNo">1960</span>            @Override<a name="line.1960"></a>
<span class="sourceLineNo">1961</span>            public void run() {<a name="line.1961"></a>
<span class="sourceLineNo">1962</span>                Role.this.checkWaitingTransactions();<a name="line.1962"></a>
<span class="sourceLineNo">1963</span>            }<a name="line.1963"></a>
<span class="sourceLineNo">1964</span>        };<a name="line.1964"></a>
<span class="sourceLineNo">1965</span>        protected final Service applyCommittedLogEntriesService = new Service(this, "apply committed logs") {<a name="line.1965"></a>
<span class="sourceLineNo">1966</span>            @Override<a name="line.1966"></a>
<span class="sourceLineNo">1967</span>            public void run() {<a name="line.1967"></a>
<span class="sourceLineNo">1968</span>                Role.this.applyCommittedLogEntries();<a name="line.1968"></a>
<span class="sourceLineNo">1969</span>            }<a name="line.1969"></a>
<span class="sourceLineNo">1970</span>        };<a name="line.1970"></a>
<span class="sourceLineNo">1971</span><a name="line.1971"></a>
<span class="sourceLineNo">1972</span>    // Constructors<a name="line.1972"></a>
<span class="sourceLineNo">1973</span><a name="line.1973"></a>
<span class="sourceLineNo">1974</span>        protected Role(RaftKVDatabase raft) {<a name="line.1974"></a>
<span class="sourceLineNo">1975</span>            this.raft = raft;<a name="line.1975"></a>
<span class="sourceLineNo">1976</span>            assert Thread.holdsLock(this.raft);<a name="line.1976"></a>
<span class="sourceLineNo">1977</span>        }<a name="line.1977"></a>
<span class="sourceLineNo">1978</span><a name="line.1978"></a>
<span class="sourceLineNo">1979</span>    // Lifecycle<a name="line.1979"></a>
<span class="sourceLineNo">1980</span><a name="line.1980"></a>
<span class="sourceLineNo">1981</span>        public void setup() {<a name="line.1981"></a>
<span class="sourceLineNo">1982</span>            assert Thread.holdsLock(this.raft);<a name="line.1982"></a>
<span class="sourceLineNo">1983</span>            this.raft.requestService(this.checkReadyTransactionsService);<a name="line.1983"></a>
<span class="sourceLineNo">1984</span>            this.raft.requestService(this.checkWaitingTransactionsService);<a name="line.1984"></a>
<span class="sourceLineNo">1985</span>            this.raft.requestService(this.applyCommittedLogEntriesService);<a name="line.1985"></a>
<span class="sourceLineNo">1986</span>        }<a name="line.1986"></a>
<span class="sourceLineNo">1987</span><a name="line.1987"></a>
<span class="sourceLineNo">1988</span>        public void shutdown() {<a name="line.1988"></a>
<span class="sourceLineNo">1989</span>            assert Thread.holdsLock(this.raft);<a name="line.1989"></a>
<span class="sourceLineNo">1990</span>            for (RaftKVTransaction tx : this.raft.openTransactions.values())<a name="line.1990"></a>
<span class="sourceLineNo">1991</span>                this.cleanupForTransaction(tx);<a name="line.1991"></a>
<span class="sourceLineNo">1992</span>        }<a name="line.1992"></a>
<span class="sourceLineNo">1993</span><a name="line.1993"></a>
<span class="sourceLineNo">1994</span>    // Service<a name="line.1994"></a>
<span class="sourceLineNo">1995</span><a name="line.1995"></a>
<span class="sourceLineNo">1996</span>        public abstract void outputQueueEmpty(String address);<a name="line.1996"></a>
<span class="sourceLineNo">1997</span><a name="line.1997"></a>
<span class="sourceLineNo">1998</span>        /**<a name="line.1998"></a>
<span class="sourceLineNo">1999</span>         * Check transactions in the {@link TxState#COMMIT_READY} state to see if we can advance them.<a name="line.1999"></a>
<span class="sourceLineNo">2000</span>         */<a name="line.2000"></a>
<span class="sourceLineNo">2001</span>        protected void checkReadyTransactions() {<a name="line.2001"></a>
<span class="sourceLineNo">2002</span>            for (RaftKVTransaction tx : new ArrayList&lt;RaftKVTransaction&gt;(this.raft.openTransactions.values()))<a name="line.2002"></a>
<span class="sourceLineNo">2003</span>                new CheckReadyTransactionService(tx).run();<a name="line.2003"></a>
<span class="sourceLineNo">2004</span>        }<a name="line.2004"></a>
<span class="sourceLineNo">2005</span><a name="line.2005"></a>
<span class="sourceLineNo">2006</span>        /**<a name="line.2006"></a>
<span class="sourceLineNo">2007</span>         * Check transactions in the {@link TxState#COMMIT_WAITING} state to see if they are committed yet.<a name="line.2007"></a>
<span class="sourceLineNo">2008</span>         * We invoke this service method whenever our {@code commitIndex} advances.<a name="line.2008"></a>
<span class="sourceLineNo">2009</span>         */<a name="line.2009"></a>
<span class="sourceLineNo">2010</span>        protected void checkWaitingTransactions() {<a name="line.2010"></a>
<span class="sourceLineNo">2011</span>            for (RaftKVTransaction tx : new ArrayList&lt;RaftKVTransaction&gt;(this.raft.openTransactions.values()))<a name="line.2011"></a>
<span class="sourceLineNo">2012</span>                new CheckWaitingTransactionService(tx).run();<a name="line.2012"></a>
<span class="sourceLineNo">2013</span>        }<a name="line.2013"></a>
<span class="sourceLineNo">2014</span><a name="line.2014"></a>
<span class="sourceLineNo">2015</span>        /**<a name="line.2015"></a>
<span class="sourceLineNo">2016</span>         * Apply committed but unapplied log entries to the state machine.<a name="line.2016"></a>
<span class="sourceLineNo">2017</span>         * We invoke this service method whenever log entries are added or our {@code commitIndex} advances.<a name="line.2017"></a>
<span class="sourceLineNo">2018</span>         */<a name="line.2018"></a>
<span class="sourceLineNo">2019</span>        protected void applyCommittedLogEntries() {<a name="line.2019"></a>
<span class="sourceLineNo">2020</span><a name="line.2020"></a>
<span class="sourceLineNo">2021</span>            // Apply committed log entries to the state machine<a name="line.2021"></a>
<span class="sourceLineNo">2022</span>            while (this.raft.lastAppliedIndex &lt; this.raft.commitIndex) {<a name="line.2022"></a>
<span class="sourceLineNo">2023</span><a name="line.2023"></a>
<span class="sourceLineNo">2024</span>                // Grab the first unwritten log entry<a name="line.2024"></a>
<span class="sourceLineNo">2025</span>                final LogEntry logEntry = this.raft.raftLog.get(0);<a name="line.2025"></a>
<span class="sourceLineNo">2026</span>                assert logEntry.getIndex() == this.raft.lastAppliedIndex + 1;<a name="line.2026"></a>
<span class="sourceLineNo">2027</span><a name="line.2027"></a>
<span class="sourceLineNo">2028</span>                // Check with subclass<a name="line.2028"></a>
<span class="sourceLineNo">2029</span>                if (!this.mayApplyLogEntry(logEntry))<a name="line.2029"></a>
<span class="sourceLineNo">2030</span>                    break;<a name="line.2030"></a>
<span class="sourceLineNo">2031</span><a name="line.2031"></a>
<span class="sourceLineNo">2032</span>                // Get the current config as of the log entry we're about to apply<a name="line.2032"></a>
<span class="sourceLineNo">2033</span>                final HashMap&lt;String, String&gt; logEntryConfig = new HashMap&lt;&gt;(this.raft.lastAppliedConfig);<a name="line.2033"></a>
<span class="sourceLineNo">2034</span>                logEntry.applyConfigChange(logEntryConfig);<a name="line.2034"></a>
<span class="sourceLineNo">2035</span><a name="line.2035"></a>
<span class="sourceLineNo">2036</span>                // Prepare combined Mutations containing prefixed log entry changes plus my own<a name="line.2036"></a>
<span class="sourceLineNo">2037</span>                final Writes logWrites = logEntry.getWrites();<a name="line.2037"></a>
<span class="sourceLineNo">2038</span>                final Writes myWrites = new Writes();<a name="line.2038"></a>
<span class="sourceLineNo">2039</span>                myWrites.getPuts().put(LAST_APPLIED_TERM_KEY, LongEncoder.encode(logEntry.getTerm()));<a name="line.2039"></a>
<span class="sourceLineNo">2040</span>                myWrites.getPuts().put(LAST_APPLIED_INDEX_KEY, LongEncoder.encode(logEntry.getIndex()));<a name="line.2040"></a>
<span class="sourceLineNo">2041</span>                myWrites.getPuts().put(LAST_APPLIED_CONFIG_KEY, this.raft.encodeConfig(logEntryConfig));<a name="line.2041"></a>
<span class="sourceLineNo">2042</span>                final Mutations mutations = new Mutations() {<a name="line.2042"></a>
<span class="sourceLineNo">2043</span><a name="line.2043"></a>
<span class="sourceLineNo">2044</span>                    @Override<a name="line.2044"></a>
<span class="sourceLineNo">2045</span>                    public Iterable&lt;KeyRange&gt; getRemoveRanges() {<a name="line.2045"></a>
<span class="sourceLineNo">2046</span>                        return Iterables.transform(logWrites.getRemoveRanges(), new PrefixKeyRangeFunction(STATE_MACHINE_PREFIX));<a name="line.2046"></a>
<span class="sourceLineNo">2047</span>                    }<a name="line.2047"></a>
<span class="sourceLineNo">2048</span><a name="line.2048"></a>
<span class="sourceLineNo">2049</span>                    @Override<a name="line.2049"></a>
<span class="sourceLineNo">2050</span>                    public Iterable&lt;Map.Entry&lt;byte[], byte[]&gt;&gt; getPutPairs() {<a name="line.2050"></a>
<span class="sourceLineNo">2051</span>                        return Iterables.concat(<a name="line.2051"></a>
<span class="sourceLineNo">2052</span>                          Iterables.transform(logWrites.getPutPairs(), new PrefixPutFunction(STATE_MACHINE_PREFIX)),<a name="line.2052"></a>
<span class="sourceLineNo">2053</span>                          myWrites.getPutPairs());<a name="line.2053"></a>
<span class="sourceLineNo">2054</span>                    }<a name="line.2054"></a>
<span class="sourceLineNo">2055</span><a name="line.2055"></a>
<span class="sourceLineNo">2056</span>                    @Override<a name="line.2056"></a>
<span class="sourceLineNo">2057</span>                    public Iterable&lt;Map.Entry&lt;byte[], Long&gt;&gt; getAdjustPairs() {<a name="line.2057"></a>
<span class="sourceLineNo">2058</span>                        return Iterables.transform(logWrites.getAdjustPairs(), new PrefixAdjustFunction(STATE_MACHINE_PREFIX));<a name="line.2058"></a>
<span class="sourceLineNo">2059</span>                    }<a name="line.2059"></a>
<span class="sourceLineNo">2060</span>                };<a name="line.2060"></a>
<span class="sourceLineNo">2061</span><a name="line.2061"></a>
<span class="sourceLineNo">2062</span>                // Apply updates to the key/value store (durably); prefix all transaction keys with STATE_MACHINE_PREFIX<a name="line.2062"></a>
<span class="sourceLineNo">2063</span>                if (this.log.isDebugEnabled())<a name="line.2063"></a>
<span class="sourceLineNo">2064</span>                    this.debug("applying committed log entry " + logEntry + " to key/value store");<a name="line.2064"></a>
<span class="sourceLineNo">2065</span>                try {<a name="line.2065"></a>
<span class="sourceLineNo">2066</span>                    this.raft.kv.mutate(mutations, true);<a name="line.2066"></a>
<span class="sourceLineNo">2067</span>                } catch (Exception e) {<a name="line.2067"></a>
<span class="sourceLineNo">2068</span>                    if (e instanceof RuntimeException &amp;&amp; e.getCause() instanceof IOException)<a name="line.2068"></a>
<span class="sourceLineNo">2069</span>                        e = (IOException)e.getCause();<a name="line.2069"></a>
<span class="sourceLineNo">2070</span>                    this.error("error applying log entry " + logEntry + " to key/value store", e);<a name="line.2070"></a>
<span class="sourceLineNo">2071</span>                    break;<a name="line.2071"></a>
<span class="sourceLineNo">2072</span>                }<a name="line.2072"></a>
<span class="sourceLineNo">2073</span><a name="line.2073"></a>
<span class="sourceLineNo">2074</span>                // Update in-memory state<a name="line.2074"></a>
<span class="sourceLineNo">2075</span>                this.raft.lastAppliedTerm = logEntry.getTerm();<a name="line.2075"></a>
<span class="sourceLineNo">2076</span>                assert logEntry.getIndex() == this.raft.lastAppliedIndex + 1;<a name="line.2076"></a>
<span class="sourceLineNo">2077</span>                this.raft.lastAppliedIndex = logEntry.getIndex();<a name="line.2077"></a>
<span class="sourceLineNo">2078</span>                logEntry.applyConfigChange(this.raft.lastAppliedConfig);<a name="line.2078"></a>
<span class="sourceLineNo">2079</span>                assert this.raft.currentConfig.equals(this.raft.buildCurrentConfig());<a name="line.2079"></a>
<span class="sourceLineNo">2080</span><a name="line.2080"></a>
<span class="sourceLineNo">2081</span>                // Delete the log entry<a name="line.2081"></a>
<span class="sourceLineNo">2082</span>                this.raft.raftLog.remove(0);<a name="line.2082"></a>
<span class="sourceLineNo">2083</span>                if (!logEntry.getFile().delete())<a name="line.2083"></a>
<span class="sourceLineNo">2084</span>                    this.error("failed to delete log file " + logEntry.getFile());<a name="line.2084"></a>
<span class="sourceLineNo">2085</span><a name="line.2085"></a>
<span class="sourceLineNo">2086</span>                // Subclass hook<a name="line.2086"></a>
<span class="sourceLineNo">2087</span>                this.logEntryApplied(logEntry);<a name="line.2087"></a>
<span class="sourceLineNo">2088</span>            }<a name="line.2088"></a>
<span class="sourceLineNo">2089</span>        }<a name="line.2089"></a>
<span class="sourceLineNo">2090</span><a name="line.2090"></a>
<span class="sourceLineNo">2091</span>        /**<a name="line.2091"></a>
<span class="sourceLineNo">2092</span>         * Determine whether the given log entry may be applied to the state machine.<a name="line.2092"></a>
<span class="sourceLineNo">2093</span>         */<a name="line.2093"></a>
<span class="sourceLineNo">2094</span>        protected boolean mayApplyLogEntry(LogEntry logEntry) {<a name="line.2094"></a>
<span class="sourceLineNo">2095</span>            return true;<a name="line.2095"></a>
<span class="sourceLineNo">2096</span>        }<a name="line.2096"></a>
<span class="sourceLineNo">2097</span><a name="line.2097"></a>
<span class="sourceLineNo">2098</span>        /**<a name="line.2098"></a>
<span class="sourceLineNo">2099</span>         * Subclass hook invoked after a log entry has been applied to the state machine.<a name="line.2099"></a>
<span class="sourceLineNo">2100</span>         */<a name="line.2100"></a>
<span class="sourceLineNo">2101</span>        protected void logEntryApplied(LogEntry logEntry) {<a name="line.2101"></a>
<span class="sourceLineNo">2102</span>        }<a name="line.2102"></a>
<span class="sourceLineNo">2103</span><a name="line.2103"></a>
<span class="sourceLineNo">2104</span>    // Transaction service classes<a name="line.2104"></a>
<span class="sourceLineNo">2105</span><a name="line.2105"></a>
<span class="sourceLineNo">2106</span>        protected abstract class AbstractTransactionService extends Service {<a name="line.2106"></a>
<span class="sourceLineNo">2107</span><a name="line.2107"></a>
<span class="sourceLineNo">2108</span>            protected final RaftKVTransaction tx;<a name="line.2108"></a>
<span class="sourceLineNo">2109</span><a name="line.2109"></a>
<span class="sourceLineNo">2110</span>            AbstractTransactionService(RaftKVTransaction tx, String desc) {<a name="line.2110"></a>
<span class="sourceLineNo">2111</span>                super(Role.this, desc);<a name="line.2111"></a>
<span class="sourceLineNo">2112</span>                assert tx != null;<a name="line.2112"></a>
<span class="sourceLineNo">2113</span>                this.tx = tx;<a name="line.2113"></a>
<span class="sourceLineNo">2114</span>            }<a name="line.2114"></a>
<span class="sourceLineNo">2115</span><a name="line.2115"></a>
<span class="sourceLineNo">2116</span>            @Override<a name="line.2116"></a>
<span class="sourceLineNo">2117</span>            public final void run() {<a name="line.2117"></a>
<span class="sourceLineNo">2118</span>                try {<a name="line.2118"></a>
<span class="sourceLineNo">2119</span>                    this.doRun();<a name="line.2119"></a>
<span class="sourceLineNo">2120</span>                } catch (KVTransactionException e) {<a name="line.2120"></a>
<span class="sourceLineNo">2121</span>                    Role.this.raft.fail(tx, e);<a name="line.2121"></a>
<span class="sourceLineNo">2122</span>                } catch (Exception e) {<a name="line.2122"></a>
<span class="sourceLineNo">2123</span>                    Role.this.raft.fail(tx, new KVTransactionException(tx, e));<a name="line.2123"></a>
<span class="sourceLineNo">2124</span>                }<a name="line.2124"></a>
<span class="sourceLineNo">2125</span>            }<a name="line.2125"></a>
<span class="sourceLineNo">2126</span><a name="line.2126"></a>
<span class="sourceLineNo">2127</span>            protected abstract void doRun();<a name="line.2127"></a>
<span class="sourceLineNo">2128</span><a name="line.2128"></a>
<span class="sourceLineNo">2129</span>            @Override<a name="line.2129"></a>
<span class="sourceLineNo">2130</span>            public boolean equals(Object obj) {<a name="line.2130"></a>
<span class="sourceLineNo">2131</span>                if (obj == null || obj.getClass() != this.getClass())<a name="line.2131"></a>
<span class="sourceLineNo">2132</span>                    return false;<a name="line.2132"></a>
<span class="sourceLineNo">2133</span>                final AbstractTransactionService that = (AbstractTransactionService)obj;<a name="line.2133"></a>
<span class="sourceLineNo">2134</span>                return this.tx.equals(that.tx);<a name="line.2134"></a>
<span class="sourceLineNo">2135</span>            }<a name="line.2135"></a>
<span class="sourceLineNo">2136</span><a name="line.2136"></a>
<span class="sourceLineNo">2137</span>            @Override<a name="line.2137"></a>
<span class="sourceLineNo">2138</span>            public int hashCode() {<a name="line.2138"></a>
<span class="sourceLineNo">2139</span>                return this.tx.hashCode();<a name="line.2139"></a>
<span class="sourceLineNo">2140</span>            }<a name="line.2140"></a>
<span class="sourceLineNo">2141</span>        }<a name="line.2141"></a>
<span class="sourceLineNo">2142</span><a name="line.2142"></a>
<span class="sourceLineNo">2143</span>        protected class CheckReadyTransactionService extends AbstractTransactionService {<a name="line.2143"></a>
<span class="sourceLineNo">2144</span><a name="line.2144"></a>
<span class="sourceLineNo">2145</span>            CheckReadyTransactionService(RaftKVTransaction tx) {<a name="line.2145"></a>
<span class="sourceLineNo">2146</span>                super(tx, "check ready tx#" + tx.getTxId());<a name="line.2146"></a>
<span class="sourceLineNo">2147</span>            }<a name="line.2147"></a>
<span class="sourceLineNo">2148</span><a name="line.2148"></a>
<span class="sourceLineNo">2149</span>            @Override<a name="line.2149"></a>
<span class="sourceLineNo">2150</span>            protected void doRun() {<a name="line.2150"></a>
<span class="sourceLineNo">2151</span>                if (this.tx.getState().equals(TxState.COMMIT_READY))<a name="line.2151"></a>
<span class="sourceLineNo">2152</span>                    Role.this.checkReadyTransaction(this.tx);<a name="line.2152"></a>
<span class="sourceLineNo">2153</span>            }<a name="line.2153"></a>
<span class="sourceLineNo">2154</span>        }<a name="line.2154"></a>
<span class="sourceLineNo">2155</span><a name="line.2155"></a>
<span class="sourceLineNo">2156</span>        protected class CheckWaitingTransactionService extends AbstractTransactionService {<a name="line.2156"></a>
<span class="sourceLineNo">2157</span><a name="line.2157"></a>
<span class="sourceLineNo">2158</span>            CheckWaitingTransactionService(RaftKVTransaction tx) {<a name="line.2158"></a>
<span class="sourceLineNo">2159</span>                super(tx, "check waiting tx#" + tx.getTxId());<a name="line.2159"></a>
<span class="sourceLineNo">2160</span>            }<a name="line.2160"></a>
<span class="sourceLineNo">2161</span><a name="line.2161"></a>
<span class="sourceLineNo">2162</span>            @Override<a name="line.2162"></a>
<span class="sourceLineNo">2163</span>            protected void doRun() {<a name="line.2163"></a>
<span class="sourceLineNo">2164</span>                if (this.tx.getState().equals(TxState.COMMIT_WAITING))<a name="line.2164"></a>
<span class="sourceLineNo">2165</span>                    Role.this.checkWaitingTransaction(this.tx);<a name="line.2165"></a>
<span class="sourceLineNo">2166</span>            }<a name="line.2166"></a>
<span class="sourceLineNo">2167</span>        }<a name="line.2167"></a>
<span class="sourceLineNo">2168</span><a name="line.2168"></a>
<span class="sourceLineNo">2169</span>    // Transactions<a name="line.2169"></a>
<span class="sourceLineNo">2170</span><a name="line.2170"></a>
<span class="sourceLineNo">2171</span>        /**<a name="line.2171"></a>
<span class="sourceLineNo">2172</span>         * Check a transaction that is ready to be committed (in the {@link TxState#COMMIT_READY} state).<a name="line.2172"></a>
<span class="sourceLineNo">2173</span>         *<a name="line.2173"></a>
<span class="sourceLineNo">2174</span>         * &lt;p&gt;<a name="line.2174"></a>
<span class="sourceLineNo">2175</span>         * This should be invoked:<a name="line.2175"></a>
<span class="sourceLineNo">2176</span>         * &lt;ul&gt;<a name="line.2176"></a>
<span class="sourceLineNo">2177</span>         *  &lt;li&gt;After changing roles&lt;/li&gt;<a name="line.2177"></a>
<span class="sourceLineNo">2178</span>         *  &lt;li&gt;After a transaction has entered the {@link TxState#COMMIT_READY} state&lt;/li&gt;<a name="line.2178"></a>
<span class="sourceLineNo">2179</span>         *  &lt;li&gt;After the leader is newly known (in {@link FollowerRole})&lt;/li&gt;<a name="line.2179"></a>
<span class="sourceLineNo">2180</span>         *  &lt;li&gt;After the leader's output queue goes from non-empty to empty (in {@link FollowerRole})&lt;/li&gt;<a name="line.2180"></a>
<span class="sourceLineNo">2181</span>         *  &lt;li&gt;After the leader's {@code commitIndex} has advanced, in case a config change transaction<a name="line.2181"></a>
<span class="sourceLineNo">2182</span>         *      is waiting on a previous config change transaction (in {@link LeaderRole})&lt;/li&gt;<a name="line.2182"></a>
<span class="sourceLineNo">2183</span>         * &lt;/ul&gt;<a name="line.2183"></a>
<span class="sourceLineNo">2184</span>         *<a name="line.2184"></a>
<span class="sourceLineNo">2185</span>         * @param tx the transaction<a name="line.2185"></a>
<span class="sourceLineNo">2186</span>         * @throws KVTransactionException if an error occurs<a name="line.2186"></a>
<span class="sourceLineNo">2187</span>         */<a name="line.2187"></a>
<span class="sourceLineNo">2188</span>        public abstract void checkReadyTransaction(RaftKVTransaction tx);<a name="line.2188"></a>
<span class="sourceLineNo">2189</span><a name="line.2189"></a>
<span class="sourceLineNo">2190</span>        /**<a name="line.2190"></a>
<span class="sourceLineNo">2191</span>         * Check a transaction waiting for its log entry to be committed (in the {@link TxState#COMMIT_WAITING} state).<a name="line.2191"></a>
<span class="sourceLineNo">2192</span>         *<a name="line.2192"></a>
<span class="sourceLineNo">2193</span>         * &lt;p&gt;<a name="line.2193"></a>
<span class="sourceLineNo">2194</span>         * This should be invoked:<a name="line.2194"></a>
<span class="sourceLineNo">2195</span>         * &lt;ul&gt;<a name="line.2195"></a>
<span class="sourceLineNo">2196</span>         *  &lt;li&gt;After changing roles&lt;/li&gt;<a name="line.2196"></a>
<span class="sourceLineNo">2197</span>         *  &lt;li&gt;After a transaction has entered the {@link TxState#COMMIT_WAITING} state&lt;/li&gt;<a name="line.2197"></a>
<span class="sourceLineNo">2198</span>         *  &lt;li&gt;After advancing my {@code commitIndex} (as leader or follower)&lt;/li&gt;<a name="line.2198"></a>
<span class="sourceLineNo">2199</span>         *  &lt;li&gt;After receiving an updated {@linkplain AppendResponse#getLeaderLeaseTimeout leader lease timeout}<a name="line.2199"></a>
<span class="sourceLineNo">2200</span>         *      (in {@link FollowerRole})&lt;/li&gt;<a name="line.2200"></a>
<span class="sourceLineNo">2201</span>         * &lt;/ul&gt;<a name="line.2201"></a>
<span class="sourceLineNo">2202</span>         *<a name="line.2202"></a>
<span class="sourceLineNo">2203</span>         * @param tx the transaction<a name="line.2203"></a>
<span class="sourceLineNo">2204</span>         * @throws KVTransactionException if an error occurs<a name="line.2204"></a>
<span class="sourceLineNo">2205</span>         */<a name="line.2205"></a>
<span class="sourceLineNo">2206</span>        private void checkWaitingTransaction(RaftKVTransaction tx) {<a name="line.2206"></a>
<span class="sourceLineNo">2207</span><a name="line.2207"></a>
<span class="sourceLineNo">2208</span>            // Handle the case the transaction's committed log index has already been applied to the state machine<a name="line.2208"></a>
<span class="sourceLineNo">2209</span>            final long commitIndex = tx.getCommitIndex();<a name="line.2209"></a>
<span class="sourceLineNo">2210</span>            if (commitIndex &lt; this.raft.lastAppliedIndex) {<a name="line.2210"></a>
<span class="sourceLineNo">2211</span><a name="line.2211"></a>
<span class="sourceLineNo">2212</span>                // This can happen if we lose contact and by the time we're back the log entry has<a name="line.2212"></a>
<span class="sourceLineNo">2213</span>                // already been applied to the state machine on some leader and that leader sent<a name="line.2213"></a>
<span class="sourceLineNo">2214</span>                // use an InstallSnapshot message. We don't know whether it actually got committed<a name="line.2214"></a>
<span class="sourceLineNo">2215</span>                // or not, so the transaction must be retried.<a name="line.2215"></a>
<span class="sourceLineNo">2216</span>                throw new RetryTransactionException(tx, "committed log entry was missed");<a name="line.2216"></a>
<span class="sourceLineNo">2217</span>            }<a name="line.2217"></a>
<span class="sourceLineNo">2218</span><a name="line.2218"></a>
<span class="sourceLineNo">2219</span>            // Has the transaction's log entry been received yet?<a name="line.2219"></a>
<span class="sourceLineNo">2220</span>            if (commitIndex &gt; this.raft.getLastLogIndex())<a name="line.2220"></a>
<span class="sourceLineNo">2221</span>                return;<a name="line.2221"></a>
<span class="sourceLineNo">2222</span><a name="line.2222"></a>
<span class="sourceLineNo">2223</span>            // Has the transaction's log entry been committed yet?<a name="line.2223"></a>
<span class="sourceLineNo">2224</span>            if (commitIndex &gt; this.raft.commitIndex)<a name="line.2224"></a>
<span class="sourceLineNo">2225</span>                return;<a name="line.2225"></a>
<span class="sourceLineNo">2226</span><a name="line.2226"></a>
<span class="sourceLineNo">2227</span>            // Verify the term of the committed log entry; if not what we expect, the log entry was overwritten by a new leader<a name="line.2227"></a>
<span class="sourceLineNo">2228</span>            final long commitTerm = tx.getCommitTerm();<a name="line.2228"></a>
<span class="sourceLineNo">2229</span>            if (this.raft.getLogTermAtIndex(commitIndex) != commitTerm)<a name="line.2229"></a>
<span class="sourceLineNo">2230</span>                throw new RetryTransactionException(tx, "leader was deposed during commit and transaction's log entry overwritten");<a name="line.2230"></a>
<span class="sourceLineNo">2231</span><a name="line.2231"></a>
<span class="sourceLineNo">2232</span>            // Check with subclass<a name="line.2232"></a>
<span class="sourceLineNo">2233</span>            if (!this.mayCommit(tx))<a name="line.2233"></a>
<span class="sourceLineNo">2234</span>                return;<a name="line.2234"></a>
<span class="sourceLineNo">2235</span><a name="line.2235"></a>
<span class="sourceLineNo">2236</span>            // Transaction is officially committed now<a name="line.2236"></a>
<span class="sourceLineNo">2237</span>            if (this.log.isTraceEnabled())<a name="line.2237"></a>
<span class="sourceLineNo">2238</span>                this.trace("commit successful for " + tx + " (commit index " + this.raft.commitIndex + " &gt;= " + commitIndex + ")");<a name="line.2238"></a>
<span class="sourceLineNo">2239</span>            this.raft.succeed(tx);<a name="line.2239"></a>
<span class="sourceLineNo">2240</span>        }<a name="line.2240"></a>
<span class="sourceLineNo">2241</span><a name="line.2241"></a>
<span class="sourceLineNo">2242</span>        protected boolean mayCommit(RaftKVTransaction tx) {<a name="line.2242"></a>
<span class="sourceLineNo">2243</span>            return true;<a name="line.2243"></a>
<span class="sourceLineNo">2244</span>        }<a name="line.2244"></a>
<span class="sourceLineNo">2245</span><a name="line.2245"></a>
<span class="sourceLineNo">2246</span>        /**<a name="line.2246"></a>
<span class="sourceLineNo">2247</span>         * Perform any role-specific transaction cleanups.<a name="line.2247"></a>
<span class="sourceLineNo">2248</span>         *<a name="line.2248"></a>
<span class="sourceLineNo">2249</span>         * &lt;p&gt;<a name="line.2249"></a>
<span class="sourceLineNo">2250</span>         * Invoked either when transaction is closed or this role is being shutdown.<a name="line.2250"></a>
<span class="sourceLineNo">2251</span>         *<a name="line.2251"></a>
<span class="sourceLineNo">2252</span>         * @param tx the transaction<a name="line.2252"></a>
<span class="sourceLineNo">2253</span>         */<a name="line.2253"></a>
<span class="sourceLineNo">2254</span>        public abstract void cleanupForTransaction(RaftKVTransaction tx);<a name="line.2254"></a>
<span class="sourceLineNo">2255</span><a name="line.2255"></a>
<span class="sourceLineNo">2256</span>        /**<a name="line.2256"></a>
<span class="sourceLineNo">2257</span>         * Check a transaction that is ready to be committed (in the {@link TxState#COMMIT_READY} state) for snapshot isolation.<a name="line.2257"></a>
<span class="sourceLineNo">2258</span>         *<a name="line.2258"></a>
<span class="sourceLineNo">2259</span>         * @param tx the transaction<a name="line.2259"></a>
<span class="sourceLineNo">2260</span>         * @return true if snapshot isolation and moved to {@link TxState@COMMIT_WAITING}<a name="line.2260"></a>
<span class="sourceLineNo">2261</span>         */<a name="line.2261"></a>
<span class="sourceLineNo">2262</span>        protected boolean checkReadyTransactionReadOnlySnapshot(RaftKVTransaction tx) {<a name="line.2262"></a>
<span class="sourceLineNo">2263</span><a name="line.2263"></a>
<span class="sourceLineNo">2264</span>            // Sanity check<a name="line.2264"></a>
<span class="sourceLineNo">2265</span>            assert Thread.holdsLock(this.raft);<a name="line.2265"></a>
<span class="sourceLineNo">2266</span>            assert tx.getState().equals(TxState.COMMIT_READY);<a name="line.2266"></a>
<span class="sourceLineNo">2267</span><a name="line.2267"></a>
<span class="sourceLineNo">2268</span>            // Check isolation<a name="line.2268"></a>
<span class="sourceLineNo">2269</span>            if (!tx.isReadOnlySnapshot())<a name="line.2269"></a>
<span class="sourceLineNo">2270</span>                return false;<a name="line.2270"></a>
<span class="sourceLineNo">2271</span><a name="line.2271"></a>
<span class="sourceLineNo">2272</span>            // For snapshot isolation, we only need to wait for the commit of the transaction's base log entry<a name="line.2272"></a>
<span class="sourceLineNo">2273</span>            tx.setCommitTerm(tx.getBaseTerm());<a name="line.2273"></a>
<span class="sourceLineNo">2274</span>            tx.setCommitIndex(tx.getBaseIndex());<a name="line.2274"></a>
<span class="sourceLineNo">2275</span>            tx.setState(TxState.COMMIT_WAITING);<a name="line.2275"></a>
<span class="sourceLineNo">2276</span>            this.raft.requestService(new CheckWaitingTransactionService(tx));<a name="line.2276"></a>
<span class="sourceLineNo">2277</span><a name="line.2277"></a>
<span class="sourceLineNo">2278</span>            // Done<a name="line.2278"></a>
<span class="sourceLineNo">2279</span>            return true;<a name="line.2279"></a>
<span class="sourceLineNo">2280</span>        }<a name="line.2280"></a>
<span class="sourceLineNo">2281</span><a name="line.2281"></a>
<span class="sourceLineNo">2282</span>    // Messages<a name="line.2282"></a>
<span class="sourceLineNo">2283</span><a name="line.2283"></a>
<span class="sourceLineNo">2284</span>        public boolean mayAdvanceCurrentTerm(Message msg) {<a name="line.2284"></a>
<span class="sourceLineNo">2285</span>            return true;<a name="line.2285"></a>
<span class="sourceLineNo">2286</span>        }<a name="line.2286"></a>
<span class="sourceLineNo">2287</span><a name="line.2287"></a>
<span class="sourceLineNo">2288</span>        protected void failUnexpectedMessage(Message msg) {<a name="line.2288"></a>
<span class="sourceLineNo">2289</span>            this.warn("rec'd unexpected message " + msg + " while in role " + this + "; ignoring");<a name="line.2289"></a>
<span class="sourceLineNo">2290</span>        }<a name="line.2290"></a>
<span class="sourceLineNo">2291</span><a name="line.2291"></a>
<span class="sourceLineNo">2292</span>    // Debug<a name="line.2292"></a>
<span class="sourceLineNo">2293</span><a name="line.2293"></a>
<span class="sourceLineNo">2294</span>        boolean checkState() {<a name="line.2294"></a>
<span class="sourceLineNo">2295</span>            return true;<a name="line.2295"></a>
<span class="sourceLineNo">2296</span>        }<a name="line.2296"></a>
<span class="sourceLineNo">2297</span><a name="line.2297"></a>
<span class="sourceLineNo">2298</span>    // Logging<a name="line.2298"></a>
<span class="sourceLineNo">2299</span><a name="line.2299"></a>
<span class="sourceLineNo">2300</span>        protected void trace(String msg, Throwable t) {<a name="line.2300"></a>
<span class="sourceLineNo">2301</span>            this.raft.trace(msg, t);<a name="line.2301"></a>
<span class="sourceLineNo">2302</span>        }<a name="line.2302"></a>
<span class="sourceLineNo">2303</span><a name="line.2303"></a>
<span class="sourceLineNo">2304</span>        protected void trace(String msg) {<a name="line.2304"></a>
<span class="sourceLineNo">2305</span>            this.raft.trace(msg);<a name="line.2305"></a>
<span class="sourceLineNo">2306</span>        }<a name="line.2306"></a>
<span class="sourceLineNo">2307</span><a name="line.2307"></a>
<span class="sourceLineNo">2308</span>        protected void debug(String msg, Throwable t) {<a name="line.2308"></a>
<span class="sourceLineNo">2309</span>            this.raft.debug(msg, t);<a name="line.2309"></a>
<span class="sourceLineNo">2310</span>        }<a name="line.2310"></a>
<span class="sourceLineNo">2311</span><a name="line.2311"></a>
<span class="sourceLineNo">2312</span>        protected void debug(String msg) {<a name="line.2312"></a>
<span class="sourceLineNo">2313</span>            this.raft.debug(msg);<a name="line.2313"></a>
<span class="sourceLineNo">2314</span>        }<a name="line.2314"></a>
<span class="sourceLineNo">2315</span><a name="line.2315"></a>
<span class="sourceLineNo">2316</span>        protected void info(String msg, Throwable t) {<a name="line.2316"></a>
<span class="sourceLineNo">2317</span>            this.raft.info(msg, t);<a name="line.2317"></a>
<span class="sourceLineNo">2318</span>        }<a name="line.2318"></a>
<span class="sourceLineNo">2319</span><a name="line.2319"></a>
<span class="sourceLineNo">2320</span>        protected void info(String msg) {<a name="line.2320"></a>
<span class="sourceLineNo">2321</span>            this.raft.info(msg);<a name="line.2321"></a>
<span class="sourceLineNo">2322</span>        }<a name="line.2322"></a>
<span class="sourceLineNo">2323</span><a name="line.2323"></a>
<span class="sourceLineNo">2324</span>        protected void warn(String msg, Throwable t) {<a name="line.2324"></a>
<span class="sourceLineNo">2325</span>            this.raft.warn(msg, t);<a name="line.2325"></a>
<span class="sourceLineNo">2326</span>        }<a name="line.2326"></a>
<span class="sourceLineNo">2327</span><a name="line.2327"></a>
<span class="sourceLineNo">2328</span>        protected void warn(String msg) {<a name="line.2328"></a>
<span class="sourceLineNo">2329</span>            this.raft.warn(msg);<a name="line.2329"></a>
<span class="sourceLineNo">2330</span>        }<a name="line.2330"></a>
<span class="sourceLineNo">2331</span><a name="line.2331"></a>
<span class="sourceLineNo">2332</span>        protected void error(String msg, Throwable t) {<a name="line.2332"></a>
<span class="sourceLineNo">2333</span>            this.raft.error(msg, t);<a name="line.2333"></a>
<span class="sourceLineNo">2334</span>        }<a name="line.2334"></a>
<span class="sourceLineNo">2335</span><a name="line.2335"></a>
<span class="sourceLineNo">2336</span>        protected void error(String msg) {<a name="line.2336"></a>
<span class="sourceLineNo">2337</span>            this.raft.error(msg);<a name="line.2337"></a>
<span class="sourceLineNo">2338</span>        }<a name="line.2338"></a>
<span class="sourceLineNo">2339</span><a name="line.2339"></a>
<span class="sourceLineNo">2340</span>    // Object<a name="line.2340"></a>
<span class="sourceLineNo">2341</span><a name="line.2341"></a>
<span class="sourceLineNo">2342</span>        @Override<a name="line.2342"></a>
<span class="sourceLineNo">2343</span>        public abstract String toString();<a name="line.2343"></a>
<span class="sourceLineNo">2344</span><a name="line.2344"></a>
<span class="sourceLineNo">2345</span>        protected String toStringPrefix() {<a name="line.2345"></a>
<span class="sourceLineNo">2346</span>            return this.getClass().getSimpleName()<a name="line.2346"></a>
<span class="sourceLineNo">2347</span>              + "[term=" + this.raft.currentTerm<a name="line.2347"></a>
<span class="sourceLineNo">2348</span>              + ",applied=" + this.raft.lastAppliedTerm + "/" + this.raft.lastAppliedIndex<a name="line.2348"></a>
<span class="sourceLineNo">2349</span>              + ",commit=" + this.raft.commitIndex<a name="line.2349"></a>
<span class="sourceLineNo">2350</span>              + ",log=" + this.raft.raftLog<a name="line.2350"></a>
<span class="sourceLineNo">2351</span>              + "]";<a name="line.2351"></a>
<span class="sourceLineNo">2352</span>        }<a name="line.2352"></a>
<span class="sourceLineNo">2353</span>    }<a name="line.2353"></a>
<span class="sourceLineNo">2354</span><a name="line.2354"></a>
<span class="sourceLineNo">2355</span>// LEADER role<a name="line.2355"></a>
<span class="sourceLineNo">2356</span><a name="line.2356"></a>
<span class="sourceLineNo">2357</span>    private static class LeaderRole extends Role {<a name="line.2357"></a>
<span class="sourceLineNo">2358</span><a name="line.2358"></a>
<span class="sourceLineNo">2359</span>        private final ArrayList&lt;LogEntry&gt; appliedLogEntries = new ArrayList&lt;&gt;(); // log entries already applied to key/value store<a name="line.2359"></a>
<span class="sourceLineNo">2360</span>        private final HashMap&lt;String, Follower&gt; followerMap = new HashMap&lt;&gt;();<a name="line.2360"></a>
<span class="sourceLineNo">2361</span>        private final Service updateLeaderCommitIndexService = new Service(this, "update leader commitIndex") {<a name="line.2361"></a>
<span class="sourceLineNo">2362</span>            @Override<a name="line.2362"></a>
<span class="sourceLineNo">2363</span>            public void run() {<a name="line.2363"></a>
<span class="sourceLineNo">2364</span>                LeaderRole.this.updateLeaderCommitIndex();<a name="line.2364"></a>
<span class="sourceLineNo">2365</span>            }<a name="line.2365"></a>
<span class="sourceLineNo">2366</span>        };<a name="line.2366"></a>
<span class="sourceLineNo">2367</span>        private final Service pruneAppliedLogEntriesService = new Service(this, "prune applied logs") {<a name="line.2367"></a>
<span class="sourceLineNo">2368</span>            @Override<a name="line.2368"></a>
<span class="sourceLineNo">2369</span>            public void run() {<a name="line.2369"></a>
<span class="sourceLineNo">2370</span>                LeaderRole.this.pruneAppliedLogEntries();<a name="line.2370"></a>
<span class="sourceLineNo">2371</span>            }<a name="line.2371"></a>
<span class="sourceLineNo">2372</span>        };<a name="line.2372"></a>
<span class="sourceLineNo">2373</span>        private final Service updateLeaseTimeoutService = new Service(this, "update lease timeout") {<a name="line.2373"></a>
<span class="sourceLineNo">2374</span>            @Override<a name="line.2374"></a>
<span class="sourceLineNo">2375</span>            public void run() {<a name="line.2375"></a>
<span class="sourceLineNo">2376</span>                LeaderRole.this.updateLeaseTimeout();<a name="line.2376"></a>
<span class="sourceLineNo">2377</span>            }<a name="line.2377"></a>
<span class="sourceLineNo">2378</span>        };<a name="line.2378"></a>
<span class="sourceLineNo">2379</span>        private final Service updateKnownFollowersService = new Service(this, "update known followers") {<a name="line.2379"></a>
<span class="sourceLineNo">2380</span>            @Override<a name="line.2380"></a>
<span class="sourceLineNo">2381</span>            public void run() {<a name="line.2381"></a>
<span class="sourceLineNo">2382</span>                LeaderRole.this.updateKnownFollowers();<a name="line.2382"></a>
<span class="sourceLineNo">2383</span>            }<a name="line.2383"></a>
<span class="sourceLineNo">2384</span>        };<a name="line.2384"></a>
<span class="sourceLineNo">2385</span>        private Timestamp leaseTimeout = new Timestamp();                   // tracks the earliest time we can be deposed<a name="line.2385"></a>
<span class="sourceLineNo">2386</span><a name="line.2386"></a>
<span class="sourceLineNo">2387</span>    // Constructors<a name="line.2387"></a>
<span class="sourceLineNo">2388</span><a name="line.2388"></a>
<span class="sourceLineNo">2389</span>        public LeaderRole(RaftKVDatabase raft) {<a name="line.2389"></a>
<span class="sourceLineNo">2390</span>            super(raft);<a name="line.2390"></a>
<span class="sourceLineNo">2391</span>        }<a name="line.2391"></a>
<span class="sourceLineNo">2392</span><a name="line.2392"></a>
<span class="sourceLineNo">2393</span>    // Lifecycle<a name="line.2393"></a>
<span class="sourceLineNo">2394</span><a name="line.2394"></a>
<span class="sourceLineNo">2395</span>        @Override<a name="line.2395"></a>
<span class="sourceLineNo">2396</span>        public void setup() {<a name="line.2396"></a>
<span class="sourceLineNo">2397</span>            super.setup();<a name="line.2397"></a>
<span class="sourceLineNo">2398</span>            this.info("entering leader role in term " + this.raft.currentTerm);<a name="line.2398"></a>
<span class="sourceLineNo">2399</span><a name="line.2399"></a>
<span class="sourceLineNo">2400</span>            // Generate follower list<a name="line.2400"></a>
<span class="sourceLineNo">2401</span>            this.updateKnownFollowers();<a name="line.2401"></a>
<span class="sourceLineNo">2402</span><a name="line.2402"></a>
<span class="sourceLineNo">2403</span>            // Append a "dummy" log entry with my current term. This allows us to advance the commit index when the last<a name="line.2403"></a>
<span class="sourceLineNo">2404</span>            // entry in our log is from a prior term. This is needed to avoid the problem where a transaction could end up<a name="line.2404"></a>
<span class="sourceLineNo">2405</span>            // waiting indefinitely for its log entry with a prior term number to be committed.<a name="line.2405"></a>
<span class="sourceLineNo">2406</span>            final LogEntry logEntry;<a name="line.2406"></a>
<span class="sourceLineNo">2407</span>            try {<a name="line.2407"></a>
<span class="sourceLineNo">2408</span>                logEntry = this.applyNewLogEntry(this.raft.new NewLogEntry(new LogEntry.Data(new Writes(), null)));<a name="line.2408"></a>
<span class="sourceLineNo">2409</span>            } catch (Exception e) {<a name="line.2409"></a>
<span class="sourceLineNo">2410</span>                this.error("error attempting to apply initial log entry", e);<a name="line.2410"></a>
<span class="sourceLineNo">2411</span>                return;<a name="line.2411"></a>
<span class="sourceLineNo">2412</span>            }<a name="line.2412"></a>
<span class="sourceLineNo">2413</span>            if (this.log.isDebugEnabled())<a name="line.2413"></a>
<span class="sourceLineNo">2414</span>                this.debug("added log entry " + logEntry + " to commit at the beginning of my new term");<a name="line.2414"></a>
<span class="sourceLineNo">2415</span>        }<a name="line.2415"></a>
<span class="sourceLineNo">2416</span><a name="line.2416"></a>
<span class="sourceLineNo">2417</span>        @Override<a name="line.2417"></a>
<span class="sourceLineNo">2418</span>        public void shutdown() {<a name="line.2418"></a>
<span class="sourceLineNo">2419</span>            super.shutdown();<a name="line.2419"></a>
<span class="sourceLineNo">2420</span>            for (Follower follower : this.followerMap.values())<a name="line.2420"></a>
<span class="sourceLineNo">2421</span>                follower.cleanup();<a name="line.2421"></a>
<span class="sourceLineNo">2422</span>        }<a name="line.2422"></a>
<span class="sourceLineNo">2423</span><a name="line.2423"></a>
<span class="sourceLineNo">2424</span>    // Service<a name="line.2424"></a>
<span class="sourceLineNo">2425</span><a name="line.2425"></a>
<span class="sourceLineNo">2426</span>        @Override<a name="line.2426"></a>
<span class="sourceLineNo">2427</span>        public void outputQueueEmpty(String address) {<a name="line.2427"></a>
<span class="sourceLineNo">2428</span><a name="line.2428"></a>
<span class="sourceLineNo">2429</span>            // Find matching follower(s) and update them if needed<a name="line.2429"></a>
<span class="sourceLineNo">2430</span>            for (Follower follower : this.followerMap.values()) {<a name="line.2430"></a>
<span class="sourceLineNo">2431</span>                if (follower.getAddress().equals(address)) {<a name="line.2431"></a>
<span class="sourceLineNo">2432</span>                    if (this.log.isTraceEnabled())<a name="line.2432"></a>
<span class="sourceLineNo">2433</span>                        this.trace("updating peer \"" + follower.getIdentity() + "\" after queue empty notification");<a name="line.2433"></a>
<span class="sourceLineNo">2434</span>                    this.raft.requestService(new UpdateFollowerService(follower));<a name="line.2434"></a>
<span class="sourceLineNo">2435</span>                }<a name="line.2435"></a>
<span class="sourceLineNo">2436</span>            }<a name="line.2436"></a>
<span class="sourceLineNo">2437</span>        }<a name="line.2437"></a>
<span class="sourceLineNo">2438</span><a name="line.2438"></a>
<span class="sourceLineNo">2439</span>        @Override<a name="line.2439"></a>
<span class="sourceLineNo">2440</span>        protected void logEntryApplied(LogEntry logEntry) {<a name="line.2440"></a>
<span class="sourceLineNo">2441</span><a name="line.2441"></a>
<span class="sourceLineNo">2442</span>            // Save log entry in the applied log entry list<a name="line.2442"></a>
<span class="sourceLineNo">2443</span>            this.appliedLogEntries.add(logEntry);<a name="line.2443"></a>
<span class="sourceLineNo">2444</span><a name="line.2444"></a>
<span class="sourceLineNo">2445</span>            // Prune applied log entries<a name="line.2445"></a>
<span class="sourceLineNo">2446</span>            this.raft.requestService(this.pruneAppliedLogEntriesService);<a name="line.2446"></a>
<span class="sourceLineNo">2447</span>        }<a name="line.2447"></a>
<span class="sourceLineNo">2448</span><a name="line.2448"></a>
<span class="sourceLineNo">2449</span>        @Override<a name="line.2449"></a>
<span class="sourceLineNo">2450</span>        protected boolean mayApplyLogEntry(LogEntry logEntry) {<a name="line.2450"></a>
<span class="sourceLineNo">2451</span><a name="line.2451"></a>
<span class="sourceLineNo">2452</span>            // Are we running out of memory? If so, go ahead.<a name="line.2452"></a>
<span class="sourceLineNo">2453</span>            final long memoryUsed = this.pruneAppliedLogEntries();<a name="line.2453"></a>
<span class="sourceLineNo">2454</span>            if (memoryUsed &gt; this.raft.maxAppliedLogMemory) {<a name="line.2454"></a>
<span class="sourceLineNo">2455</span>                if (this.log.isTraceEnabled()) {<a name="line.2455"></a>
<span class="sourceLineNo">2456</span>                    this.trace("allowing log entry " + logEntry + " to be applied because memory usage is high: "<a name="line.2456"></a>
<span class="sourceLineNo">2457</span>                      + memoryUsed + " &gt; " + this.raft.maxAppliedLogMemory);<a name="line.2457"></a>
<span class="sourceLineNo">2458</span>                }<a name="line.2458"></a>
<span class="sourceLineNo">2459</span>                return true;<a name="line.2459"></a>
<span class="sourceLineNo">2460</span>            }<a name="line.2460"></a>
<span class="sourceLineNo">2461</span><a name="line.2461"></a>
<span class="sourceLineNo">2462</span>            // If any snapshots are in progress, we don't want to apply any log entries with index greater than the snapshot's<a name="line.2462"></a>
<span class="sourceLineNo">2463</span>            // index, because then we'd "lose" the ability to update the follower with that log entry, and as a result just have<a name="line.2463"></a>
<span class="sourceLineNo">2464</span>            // to send a snapshot again. However, we impose a limit on how long we'll wait for a slow follower.<a name="line.2464"></a>
<span class="sourceLineNo">2465</span>            for (Follower follower : this.followerMap.values()) {<a name="line.2465"></a>
<span class="sourceLineNo">2466</span>                final SnapshotTransmit snapshotTransmit = follower.getSnapshotTransmit();<a name="line.2466"></a>
<span class="sourceLineNo">2467</span>                if (snapshotTransmit != null<a name="line.2467"></a>
<span class="sourceLineNo">2468</span>                  &amp;&amp; snapshotTransmit.getSnapshotIndex() &lt; logEntry.getIndex()              // currently, this will always be true<a name="line.2468"></a>
<span class="sourceLineNo">2469</span>                  &amp;&amp; snapshotTransmit.getAge() &lt; MAX_SNAPSHOT_TRANSMIT_AGE) {<a name="line.2469"></a>
<span class="sourceLineNo">2470</span>                    if (this.log.isTraceEnabled()) {<a name="line.2470"></a>
<span class="sourceLineNo">2471</span>                        this.trace("delaying application of " + logEntry + " because of in-progress snapshot install of "<a name="line.2471"></a>
<span class="sourceLineNo">2472</span>                          + snapshotTransmit.getSnapshotTerm() + "/" + snapshotTransmit.getSnapshotIndex()<a name="line.2472"></a>
<span class="sourceLineNo">2473</span>                          + " to " + follower);<a name="line.2473"></a>
<span class="sourceLineNo">2474</span>                    }<a name="line.2474"></a>
<span class="sourceLineNo">2475</span>                    return false;<a name="line.2475"></a>
<span class="sourceLineNo">2476</span>                }<a name="line.2476"></a>
<span class="sourceLineNo">2477</span>            }<a name="line.2477"></a>
<span class="sourceLineNo">2478</span><a name="line.2478"></a>
<span class="sourceLineNo">2479</span>            // If some follower does not yet have the log entry, wait for them to get it (up to some maximum time)<a name="line.2479"></a>
<span class="sourceLineNo">2480</span>            if (logEntry.getAge() &lt; MAX_SLOW_FOLLOWER_APPLY_DELAY_HEARTBEATS * this.raft.heartbeatTimeout) {<a name="line.2480"></a>
<span class="sourceLineNo">2481</span>                for (Follower follower : this.followerMap.values()) {<a name="line.2481"></a>
<span class="sourceLineNo">2482</span>                    if (follower.getMatchIndex() &lt; logEntry.getIndex()) {<a name="line.2482"></a>
<span class="sourceLineNo">2483</span>                        if (this.log.isTraceEnabled()) {<a name="line.2483"></a>
<span class="sourceLineNo">2484</span>                            this.trace("delaying application of " + logEntry + " (age " + logEntry.getAge()<a name="line.2484"></a>
<span class="sourceLineNo">2485</span>                              + " &lt; " + (MAX_SLOW_FOLLOWER_APPLY_DELAY_HEARTBEATS * this.raft.heartbeatTimeout)<a name="line.2485"></a>
<span class="sourceLineNo">2486</span>                              + ") because of slow " + follower);<a name="line.2486"></a>
<span class="sourceLineNo">2487</span>                        }<a name="line.2487"></a>
<span class="sourceLineNo">2488</span>                        return false;<a name="line.2488"></a>
<span class="sourceLineNo">2489</span>                    }<a name="line.2489"></a>
<span class="sourceLineNo">2490</span>                }<a name="line.2490"></a>
<span class="sourceLineNo">2491</span>            }<a name="line.2491"></a>
<span class="sourceLineNo">2492</span><a name="line.2492"></a>
<span class="sourceLineNo">2493</span>            // OK<a name="line.2493"></a>
<span class="sourceLineNo">2494</span>            return true;<a name="line.2494"></a>
<span class="sourceLineNo">2495</span>        }<a name="line.2495"></a>
<span class="sourceLineNo">2496</span><a name="line.2496"></a>
<span class="sourceLineNo">2497</span>        /**<a name="line.2497"></a>
<span class="sourceLineNo">2498</span>         * Check memory usage for both unapplied and applied log entries, and discard already-applied log entries as appropriate<a name="line.2498"></a>
<span class="sourceLineNo">2499</span>         * to stay under the memory limits.<a name="line.2499"></a>
<span class="sourceLineNo">2500</span>         *<a name="line.2500"></a>
<span class="sourceLineNo">2501</span>         * &lt;p&gt;<a name="line.2501"></a>
<span class="sourceLineNo">2502</span>         * This should be invoked:<a name="line.2502"></a>
<span class="sourceLineNo">2503</span>         * &lt;ul&gt;<a name="line.2503"></a>
<span class="sourceLineNo">2504</span>         *  &lt;li&gt;After a log entry has been added to the log&lt;/li&gt;<a name="line.2504"></a>
<span class="sourceLineNo">2505</span>         *  &lt;li&gt;After a log entry has been applied to the state machine&lt;/li&gt;<a name="line.2505"></a>
<span class="sourceLineNo">2506</span>         * &lt;/ul&gt;<a name="line.2506"></a>
<span class="sourceLineNo">2507</span>         *<a name="line.2507"></a>
<span class="sourceLineNo">2508</span>         * @return total applied log entry memory used (approximate)<a name="line.2508"></a>
<span class="sourceLineNo">2509</span>         */<a name="line.2509"></a>
<span class="sourceLineNo">2510</span>        private long pruneAppliedLogEntries() {<a name="line.2510"></a>
<span class="sourceLineNo">2511</span><a name="line.2511"></a>
<span class="sourceLineNo">2512</span>            // Calculate total memory usage<a name="line.2512"></a>
<span class="sourceLineNo">2513</span>            long totalLogEntryMemoryUsed = 0;<a name="line.2513"></a>
<span class="sourceLineNo">2514</span>            for (LogEntry logEntry : Iterables.concat(this.appliedLogEntries, this.raft.raftLog))<a name="line.2514"></a>
<span class="sourceLineNo">2515</span>                totalLogEntryMemoryUsed += logEntry.getFileSize();<a name="line.2515"></a>
<span class="sourceLineNo">2516</span><a name="line.2516"></a>
<span class="sourceLineNo">2517</span>            // Delete applied log entries to stay under limits<a name="line.2517"></a>
<span class="sourceLineNo">2518</span>            long numAppliedLogEntries = this.appliedLogEntries.size();<a name="line.2518"></a>
<span class="sourceLineNo">2519</span>            for (Iterator&lt;LogEntry&gt; i = this.appliedLogEntries.iterator(); i.hasNext(); ) {<a name="line.2519"></a>
<span class="sourceLineNo">2520</span>                final LogEntry logEntry = i.next();<a name="line.2520"></a>
<span class="sourceLineNo">2521</span>                if (logEntry.getAge() &gt;= this.raft.maxTransactionDuration<a name="line.2521"></a>
<span class="sourceLineNo">2522</span>                  || totalLogEntryMemoryUsed &gt; this.raft.maxAppliedLogMemory<a name="line.2522"></a>
<span class="sourceLineNo">2523</span>                  || numAppliedLogEntries &gt; MAX_APPLIED_LOG_ENTRIES) {<a name="line.2523"></a>
<span class="sourceLineNo">2524</span>                    i.remove();<a name="line.2524"></a>
<span class="sourceLineNo">2525</span>                    totalLogEntryMemoryUsed -= logEntry.getFileSize();<a name="line.2525"></a>
<span class="sourceLineNo">2526</span>                    numAppliedLogEntries--;<a name="line.2526"></a>
<span class="sourceLineNo">2527</span>                }<a name="line.2527"></a>
<span class="sourceLineNo">2528</span>            }<a name="line.2528"></a>
<span class="sourceLineNo">2529</span><a name="line.2529"></a>
<span class="sourceLineNo">2530</span>            // Done<a name="line.2530"></a>
<span class="sourceLineNo">2531</span>            return totalLogEntryMemoryUsed;<a name="line.2531"></a>
<span class="sourceLineNo">2532</span>        }<a name="line.2532"></a>
<span class="sourceLineNo">2533</span><a name="line.2533"></a>
<span class="sourceLineNo">2534</span>        /**<a name="line.2534"></a>
<span class="sourceLineNo">2535</span>         * Update my {@code commitIndex} based on followers' {@code matchIndex}'s.<a name="line.2535"></a>
<span class="sourceLineNo">2536</span>         *<a name="line.2536"></a>
<span class="sourceLineNo">2537</span>         * &lt;p&gt;<a name="line.2537"></a>
<span class="sourceLineNo">2538</span>         * This should be invoked:<a name="line.2538"></a>
<span class="sourceLineNo">2539</span>         * &lt;ul&gt;<a name="line.2539"></a>
<span class="sourceLineNo">2540</span>         *  &lt;li&gt;After any log entry has been added to the log, if we have zero followers&lt;/li&gt;<a name="line.2540"></a>
<span class="sourceLineNo">2541</span>         *  &lt;li&gt;After a log entry that contains a configuration change has been added to the log&lt;/li&gt;<a name="line.2541"></a>
<span class="sourceLineNo">2542</span>         *  &lt;li&gt;After a follower's {@linkplain Follower#getMatchIndex match index} has advanced&lt;/li&gt;<a name="line.2542"></a>
<span class="sourceLineNo">2543</span>         * &lt;/ul&gt;<a name="line.2543"></a>
<span class="sourceLineNo">2544</span>         */<a name="line.2544"></a>
<span class="sourceLineNo">2545</span>        private void updateLeaderCommitIndex() {<a name="line.2545"></a>
<span class="sourceLineNo">2546</span><a name="line.2546"></a>
<span class="sourceLineNo">2547</span>            // Update my commit index based on when a majority of cluster members have ack'd log entries<a name="line.2547"></a>
<span class="sourceLineNo">2548</span>            final long lastLogIndex = this.raft.getLastLogIndex();<a name="line.2548"></a>
<span class="sourceLineNo">2549</span>            while (this.raft.commitIndex &lt; lastLogIndex) {<a name="line.2549"></a>
<span class="sourceLineNo">2550</span><a name="line.2550"></a>
<span class="sourceLineNo">2551</span>                // Get the index in question<a name="line.2551"></a>
<span class="sourceLineNo">2552</span>                final long index = this.raft.commitIndex + 1;<a name="line.2552"></a>
<span class="sourceLineNo">2553</span><a name="line.2553"></a>
<span class="sourceLineNo">2554</span>                // Count the number of nodes which have a copy of the log entry at index<a name="line.2554"></a>
<span class="sourceLineNo">2555</span>                int numVotes = this.raft.isClusterMember() ? 1 : 0;                         // count myself, if member<a name="line.2555"></a>
<span class="sourceLineNo">2556</span>                for (Follower follower : this.followerMap.values()) {                       // count followers who are members<a name="line.2556"></a>
<span class="sourceLineNo">2557</span>                    if (follower.getMatchIndex() &gt;= index &amp;&amp; this.raft.isClusterMember(follower.getIdentity()))<a name="line.2557"></a>
<span class="sourceLineNo">2558</span>                        numVotes++;<a name="line.2558"></a>
<span class="sourceLineNo">2559</span>                }<a name="line.2559"></a>
<span class="sourceLineNo">2560</span><a name="line.2560"></a>
<span class="sourceLineNo">2561</span>                // Do a majority of cluster nodes have a copy of this log entry?<a name="line.2561"></a>
<span class="sourceLineNo">2562</span>                final int majority = this.raft.currentConfig.size() / 2 + 1;<a name="line.2562"></a>
<span class="sourceLineNo">2563</span>                if (numVotes &lt; majority)<a name="line.2563"></a>
<span class="sourceLineNo">2564</span>                    break;<a name="line.2564"></a>
<span class="sourceLineNo">2565</span><a name="line.2565"></a>
<span class="sourceLineNo">2566</span>                // Log entry term must match my current term<a name="line.2566"></a>
<span class="sourceLineNo">2567</span>                final LogEntry logEntry = this.raft.getLogEntryAtIndex(index);<a name="line.2567"></a>
<span class="sourceLineNo">2568</span>                if (logEntry.getTerm() != this.raft.currentTerm)<a name="line.2568"></a>
<span class="sourceLineNo">2569</span>                    break;<a name="line.2569"></a>
<span class="sourceLineNo">2570</span><a name="line.2570"></a>
<span class="sourceLineNo">2571</span>                // Update commit index<a name="line.2571"></a>
<span class="sourceLineNo">2572</span>                if (this.log.isDebugEnabled()) {<a name="line.2572"></a>
<span class="sourceLineNo">2573</span>                    this.debug("advancing commit index from " + this.raft.commitIndex + " -&gt; " + index<a name="line.2573"></a>
<span class="sourceLineNo">2574</span>                      + " based on " + numVotes + "/" + this.raft.currentConfig.size() + " nodes having received " + logEntry);<a name="line.2574"></a>
<span class="sourceLineNo">2575</span>                }<a name="line.2575"></a>
<span class="sourceLineNo">2576</span>                this.raft.commitIndex = index;<a name="line.2576"></a>
<span class="sourceLineNo">2577</span><a name="line.2577"></a>
<span class="sourceLineNo">2578</span>                // Perform various service<a name="line.2578"></a>
<span class="sourceLineNo">2579</span>                this.raft.requestService(this.checkReadyTransactionsService);<a name="line.2579"></a>
<span class="sourceLineNo">2580</span>                this.raft.requestService(this.checkWaitingTransactionsService);<a name="line.2580"></a>
<span class="sourceLineNo">2581</span>                this.raft.requestService(this.applyCommittedLogEntriesService);<a name="line.2581"></a>
<span class="sourceLineNo">2582</span><a name="line.2582"></a>
<span class="sourceLineNo">2583</span>                // Notify all followers with the updated leaderCommit<a name="line.2583"></a>
<span class="sourceLineNo">2584</span>                this.updateAllFollowersNow();<a name="line.2584"></a>
<span class="sourceLineNo">2585</span>            }<a name="line.2585"></a>
<span class="sourceLineNo">2586</span><a name="line.2586"></a>
<span class="sourceLineNo">2587</span>            // If we are no longer a member of our cluster, step down as leader after the latest config change is committed<a name="line.2587"></a>
<span class="sourceLineNo">2588</span>            if (!this.raft.isClusterMember()<a name="line.2588"></a>
<span class="sourceLineNo">2589</span>              &amp;&amp; this.raft.commitIndex &gt;= this.findMostRecentConfigChangeMatching(Predicates.&lt;String[]&gt;alwaysTrue())) {<a name="line.2589"></a>
<span class="sourceLineNo">2590</span>                this.log.info("stepping down as leader of cluster (no longer a member)");<a name="line.2590"></a>
<span class="sourceLineNo">2591</span>                this.raft.changeRole(new FollowerRole(this.raft));<a name="line.2591"></a>
<span class="sourceLineNo">2592</span>            }<a name="line.2592"></a>
<span class="sourceLineNo">2593</span>        }<a name="line.2593"></a>
<span class="sourceLineNo">2594</span><a name="line.2594"></a>
<span class="sourceLineNo">2595</span>        /**<a name="line.2595"></a>
<span class="sourceLineNo">2596</span>         * Update my {@code leaseTimeout} based on followers' returned {@code leaderTimeout}'s.<a name="line.2596"></a>
<span class="sourceLineNo">2597</span>         *<a name="line.2597"></a>
<span class="sourceLineNo">2598</span>         * &lt;p&gt;<a name="line.2598"></a>
<span class="sourceLineNo">2599</span>         * This should be invoked:<a name="line.2599"></a>
<span class="sourceLineNo">2600</span>         * &lt;ul&gt;<a name="line.2600"></a>
<span class="sourceLineNo">2601</span>         *  &lt;li&gt;After a follower has replied with an {@link AppendResponse} containing a newer<a name="line.2601"></a>
<span class="sourceLineNo">2602</span>         *      {@linkplain AppendResponse#getLeaderTimestamp leader timestamp} than before&lt;/li&gt;<a name="line.2602"></a>
<span class="sourceLineNo">2603</span>         * &lt;/ul&gt;<a name="line.2603"></a>
<span class="sourceLineNo">2604</span>         */<a name="line.2604"></a>
<span class="sourceLineNo">2605</span>        private void updateLeaseTimeout() {<a name="line.2605"></a>
<span class="sourceLineNo">2606</span><a name="line.2606"></a>
<span class="sourceLineNo">2607</span>            // Only needed when we have followers<a name="line.2607"></a>
<span class="sourceLineNo">2608</span>            final int numFollowers = this.followerMap.size();<a name="line.2608"></a>
<span class="sourceLineNo">2609</span>            if (numFollowers == 0)<a name="line.2609"></a>
<span class="sourceLineNo">2610</span>                return;<a name="line.2610"></a>
<span class="sourceLineNo">2611</span><a name="line.2611"></a>
<span class="sourceLineNo">2612</span>            // Get all cluster member leader timestamps, sorted in increasing order<a name="line.2612"></a>
<span class="sourceLineNo">2613</span>            final Timestamp[] leaderTimestamps = new Timestamp[this.raft.currentConfig.size()];<a name="line.2613"></a>
<span class="sourceLineNo">2614</span>            int index = 0;<a name="line.2614"></a>
<span class="sourceLineNo">2615</span>            if (this.raft.isClusterMember())<a name="line.2615"></a>
<span class="sourceLineNo">2616</span>                leaderTimestamps[index++] = new Timestamp();                                // only matters in single node cluster<a name="line.2616"></a>
<span class="sourceLineNo">2617</span>            for (Follower follower : this.followerMap.values()) {<a name="line.2617"></a>
<span class="sourceLineNo">2618</span>                if (this.raft.isClusterMember(follower.getIdentity()))<a name="line.2618"></a>
<span class="sourceLineNo">2619</span>                    leaderTimestamps[index++] = follower.getLeaderTimestamp();<a name="line.2619"></a>
<span class="sourceLineNo">2620</span>            }<a name="line.2620"></a>
<span class="sourceLineNo">2621</span>            Preconditions.checkArgument(index == leaderTimestamps.length);                  // sanity check<a name="line.2621"></a>
<span class="sourceLineNo">2622</span>            Arrays.sort(leaderTimestamps);<a name="line.2622"></a>
<span class="sourceLineNo">2623</span><a name="line.2623"></a>
<span class="sourceLineNo">2624</span>            //<a name="line.2624"></a>
<span class="sourceLineNo">2625</span>            // Calculate highest leaderTimeout shared by a majority of cluster members, based on sorted array:<a name="line.2625"></a>
<span class="sourceLineNo">2626</span>            //<a name="line.2626"></a>
<span class="sourceLineNo">2627</span>            //  # nodes    timestamps<a name="line.2627"></a>
<span class="sourceLineNo">2628</span>            //  -------    ----------<a name="line.2628"></a>
<span class="sourceLineNo">2629</span>            //     5       [ ][ ][x][x][x]        3/5 x's make a majority at index (5 - 1)/2 = 2<a name="line.2629"></a>
<span class="sourceLineNo">2630</span>            //     6       [ ][ ][x][x][x][x]     4/6 x's make a majority at index (6 - 1)/2 = 2<a name="line.2630"></a>
<span class="sourceLineNo">2631</span>            //<a name="line.2631"></a>
<span class="sourceLineNo">2632</span>            // The minimum leaderTimeout shared by a majority of nodes is at index (leaderTimestamps.length - 1) / 2.<a name="line.2632"></a>
<span class="sourceLineNo">2633</span>            // We then add the minimum election timeout, then subtract a little for clock drift.<a name="line.2633"></a>
<span class="sourceLineNo">2634</span>            //<a name="line.2634"></a>
<span class="sourceLineNo">2635</span>            final Timestamp newLeaseTimeout = leaderTimestamps[(leaderTimestamps.length + 1) / 2]<a name="line.2635"></a>
<span class="sourceLineNo">2636</span>              .offset((int)(this.raft.minElectionTimeout * (1.0f - MAX_CLOCK_DRIFT)));<a name="line.2636"></a>
<span class="sourceLineNo">2637</span><a name="line.2637"></a>
<span class="sourceLineNo">2638</span>            // Immediately notify any followers who are waiting on this updated timestamp (or any smaller timestamp)<a name="line.2638"></a>
<span class="sourceLineNo">2639</span>            if (newLeaseTimeout.compareTo(this.leaseTimeout) &gt; 0) {<a name="line.2639"></a>
<span class="sourceLineNo">2640</span><a name="line.2640"></a>
<span class="sourceLineNo">2641</span>                // Update my leader lease timeout<a name="line.2641"></a>
<span class="sourceLineNo">2642</span>                if (this.log.isTraceEnabled())<a name="line.2642"></a>
<span class="sourceLineNo">2643</span>                    this.trace("updating my lease timeout from " + this.leaseTimeout + " -&gt; " + newLeaseTimeout);<a name="line.2643"></a>
<span class="sourceLineNo">2644</span>                this.leaseTimeout = newLeaseTimeout;<a name="line.2644"></a>
<span class="sourceLineNo">2645</span><a name="line.2645"></a>
<span class="sourceLineNo">2646</span>                // Notify any followers who care<a name="line.2646"></a>
<span class="sourceLineNo">2647</span>                for (Follower follower : this.followerMap.values()) {<a name="line.2647"></a>
<span class="sourceLineNo">2648</span>                    final NavigableSet&lt;Timestamp&gt; timeouts = follower.getCommitLeaseTimeouts().headSet(this.leaseTimeout, true);<a name="line.2648"></a>
<span class="sourceLineNo">2649</span>                    if (!timeouts.isEmpty()) {<a name="line.2649"></a>
<span class="sourceLineNo">2650</span>                        follower.updateNow();                           // notify follower so it can commit waiting transaction(s)<a name="line.2650"></a>
<span class="sourceLineNo">2651</span>                        timeouts.clear();<a name="line.2651"></a>
<span class="sourceLineNo">2652</span>                    }<a name="line.2652"></a>
<span class="sourceLineNo">2653</span>                }<a name="line.2653"></a>
<span class="sourceLineNo">2654</span>            }<a name="line.2654"></a>
<span class="sourceLineNo">2655</span>        }<a name="line.2655"></a>
<span class="sourceLineNo">2656</span><a name="line.2656"></a>
<span class="sourceLineNo">2657</span>        /**<a name="line.2657"></a>
<span class="sourceLineNo">2658</span>         * Update our list of followers to match our current configuration.<a name="line.2658"></a>
<span class="sourceLineNo">2659</span>         *<a name="line.2659"></a>
<span class="sourceLineNo">2660</span>         * &lt;p&gt;<a name="line.2660"></a>
<span class="sourceLineNo">2661</span>         * This should be invoked:<a name="line.2661"></a>
<span class="sourceLineNo">2662</span>         * &lt;ul&gt;<a name="line.2662"></a>
<span class="sourceLineNo">2663</span>         *  &lt;li&gt;After a log entry that contains a configuration change has been added to the log&lt;/li&gt;<a name="line.2663"></a>
<span class="sourceLineNo">2664</span>         *  &lt;li&gt;When the {@linkplain Follower#getNextIndex next index} of a follower not in the current config advances&lt;/li&gt;<a name="line.2664"></a>
<span class="sourceLineNo">2665</span>         * &lt;/ul&gt;<a name="line.2665"></a>
<span class="sourceLineNo">2666</span>         */<a name="line.2666"></a>
<span class="sourceLineNo">2667</span>        private void updateKnownFollowers() {<a name="line.2667"></a>
<span class="sourceLineNo">2668</span><a name="line.2668"></a>
<span class="sourceLineNo">2669</span>            // Compare known followers with the current config and determine who needs to be be added or removed<a name="line.2669"></a>
<span class="sourceLineNo">2670</span>            final HashSet&lt;String&gt; adds = new HashSet&lt;&gt;(this.raft.currentConfig.keySet());<a name="line.2670"></a>
<span class="sourceLineNo">2671</span>            adds.removeAll(this.followerMap.keySet());<a name="line.2671"></a>
<span class="sourceLineNo">2672</span>            adds.remove(this.raft.identity);<a name="line.2672"></a>
<span class="sourceLineNo">2673</span>            final HashSet&lt;String&gt; dels = new HashSet&lt;&gt;(this.followerMap.keySet());<a name="line.2673"></a>
<span class="sourceLineNo">2674</span>            dels.removeAll(this.raft.currentConfig.keySet());<a name="line.2674"></a>
<span class="sourceLineNo">2675</span><a name="line.2675"></a>
<span class="sourceLineNo">2676</span>            // Keep around a follower after its removal until it receives the config change that removed it<a name="line.2676"></a>
<span class="sourceLineNo">2677</span>            for (Follower follower : this.followerMap.values()) {<a name="line.2677"></a>
<span class="sourceLineNo">2678</span><a name="line.2678"></a>
<span class="sourceLineNo">2679</span>                // Is this follower scheduled for deletion?<a name="line.2679"></a>
<span class="sourceLineNo">2680</span>                final String peer = follower.getIdentity();<a name="line.2680"></a>
<span class="sourceLineNo">2681</span>                if (!dels.contains(peer))<a name="line.2681"></a>
<span class="sourceLineNo">2682</span>                    continue;<a name="line.2682"></a>
<span class="sourceLineNo">2683</span><a name="line.2683"></a>
<span class="sourceLineNo">2684</span>                // Find the most recent log entry containing a config change in which the follower was removed<a name="line.2684"></a>
<span class="sourceLineNo">2685</span>                final String node = follower.getIdentity();<a name="line.2685"></a>
<span class="sourceLineNo">2686</span>                final long index = this.findMostRecentConfigChangeMatching(new Predicate&lt;String[]&gt;() {<a name="line.2686"></a>
<span class="sourceLineNo">2687</span>                    @Override<a name="line.2687"></a>
<span class="sourceLineNo">2688</span>                    public boolean apply(String[] configChange) {<a name="line.2688"></a>
<span class="sourceLineNo">2689</span>                        return configChange[0].equals(node) &amp;&amp; configChange[1] == null;<a name="line.2689"></a>
<span class="sourceLineNo">2690</span>                    }<a name="line.2690"></a>
<span class="sourceLineNo">2691</span>                });<a name="line.2691"></a>
<span class="sourceLineNo">2692</span><a name="line.2692"></a>
<span class="sourceLineNo">2693</span>                // If follower has not received that log entry yet, keep on updating them until they do<a name="line.2693"></a>
<span class="sourceLineNo">2694</span>                if (follower.getMatchIndex() &lt; index)<a name="line.2694"></a>
<span class="sourceLineNo">2695</span>                    dels.remove(peer);<a name="line.2695"></a>
<span class="sourceLineNo">2696</span>            }<a name="line.2696"></a>
<span class="sourceLineNo">2697</span><a name="line.2697"></a>
<span class="sourceLineNo">2698</span>            // Add new followers<a name="line.2698"></a>
<span class="sourceLineNo">2699</span>            for (String peer : adds) {<a name="line.2699"></a>
<span class="sourceLineNo">2700</span>                final String address = this.raft.currentConfig.get(peer);<a name="line.2700"></a>
<span class="sourceLineNo">2701</span>                final Follower follower = new Follower(peer, address, this.raft.getLastLogIndex());<a name="line.2701"></a>
<span class="sourceLineNo">2702</span>                this.debug("adding new follower \"" + peer + "\" at " + address);<a name="line.2702"></a>
<span class="sourceLineNo">2703</span>                follower.setUpdateTimer(<a name="line.2703"></a>
<span class="sourceLineNo">2704</span>                  this.raft.new Timer("update timer for \"" + peer + "\"", new UpdateFollowerService(follower)));<a name="line.2704"></a>
<span class="sourceLineNo">2705</span>                this.followerMap.put(peer, follower);<a name="line.2705"></a>
<span class="sourceLineNo">2706</span>                follower.updateNow();                                               // schedule an immediate update<a name="line.2706"></a>
<span class="sourceLineNo">2707</span>            }<a name="line.2707"></a>
<span class="sourceLineNo">2708</span><a name="line.2708"></a>
<span class="sourceLineNo">2709</span>            // Remove old followers<a name="line.2709"></a>
<span class="sourceLineNo">2710</span>            for (String peer : dels) {<a name="line.2710"></a>
<span class="sourceLineNo">2711</span>                final Follower follower = this.followerMap.remove(peer);<a name="line.2711"></a>
<span class="sourceLineNo">2712</span>                this.debug("removing old follower \"" + peer + "\"");<a name="line.2712"></a>
<span class="sourceLineNo">2713</span>                follower.cleanup();<a name="line.2713"></a>
<span class="sourceLineNo">2714</span>            }<a name="line.2714"></a>
<span class="sourceLineNo">2715</span>        }<a name="line.2715"></a>
<span class="sourceLineNo">2716</span><a name="line.2716"></a>
<span class="sourceLineNo">2717</span>        /**<a name="line.2717"></a>
<span class="sourceLineNo">2718</span>         * Check whether a follower needs an update and send one if so.<a name="line.2718"></a>
<span class="sourceLineNo">2719</span>         *<a name="line.2719"></a>
<span class="sourceLineNo">2720</span>         * &lt;p&gt;<a name="line.2720"></a>
<span class="sourceLineNo">2721</span>         * This should be invoked:<a name="line.2721"></a>
<span class="sourceLineNo">2722</span>         * &lt;ul&gt;<a name="line.2722"></a>
<span class="sourceLineNo">2723</span>         *  &lt;li&gt;After a new follower has been added&lt;/li&gt;<a name="line.2723"></a>
<span class="sourceLineNo">2724</span>         *  &lt;li&gt;When the output queue for a follower goes from non-empty to empty&lt;/li&gt;<a name="line.2724"></a>
<span class="sourceLineNo">2725</span>         *  &lt;li&gt;After the follower's {@linkplain Follower#getUpdateTimer update timer} has expired&lt;/li&gt;<a name="line.2725"></a>
<span class="sourceLineNo">2726</span>         *  &lt;li&gt;After a new log entry has been added to the log (all followers)&lt;/li&gt;<a name="line.2726"></a>
<span class="sourceLineNo">2727</span>         *  &lt;li&gt;After receiving an {@link AppendResponse} that caused the follower's<a name="line.2727"></a>
<span class="sourceLineNo">2728</span>         *      {@linkplain Follower#getNextIndex next index} to change&lt;/li&gt;<a name="line.2728"></a>
<span class="sourceLineNo">2729</span>         *  &lt;li&gt;After receiving the first positive {@link AppendResponse} to a probe&lt;/li&gt;<a name="line.2729"></a>
<span class="sourceLineNo">2730</span>         *  &lt;li&gt;After our {@code commitIndex} has advanced (all followers)&lt;/li&gt;<a name="line.2730"></a>
<span class="sourceLineNo">2731</span>         *  &lt;li&gt;After our {@code leaseTimeout} has advanced past one or more of a follower's<a name="line.2731"></a>
<span class="sourceLineNo">2732</span>         *      {@linkplain Follower#getCommitLeaseTimeouts commit lease timeouts} (with update timer reset)&lt;/li&gt;<a name="line.2732"></a>
<span class="sourceLineNo">2733</span>         *  &lt;li&gt;After sending a {@link CommitResponse} with a non-null {@linkplain CommitResponse#getCommitLeaderLeaseTimeout<a name="line.2733"></a>
<span class="sourceLineNo">2734</span>         *      commit leader lease timeout} (all followers) to probe for updated leader timestamps&lt;/li&gt;<a name="line.2734"></a>
<span class="sourceLineNo">2735</span>         *  &lt;li&gt;After starting, aborting, or completing a snapshot install for a follower&lt;/li&gt;<a name="line.2735"></a>
<span class="sourceLineNo">2736</span>         * &lt;/ul&gt;<a name="line.2736"></a>
<span class="sourceLineNo">2737</span>         */<a name="line.2737"></a>
<span class="sourceLineNo">2738</span>        private void updateFollower(Follower follower) {<a name="line.2738"></a>
<span class="sourceLineNo">2739</span><a name="line.2739"></a>
<span class="sourceLineNo">2740</span>            // If follower has an in-progress snapshot that has become too stale, abort it<a name="line.2740"></a>
<span class="sourceLineNo">2741</span>            final String peer = follower.getIdentity();<a name="line.2741"></a>
<span class="sourceLineNo">2742</span>            SnapshotTransmit snapshotTransmit = follower.getSnapshotTransmit();<a name="line.2742"></a>
<span class="sourceLineNo">2743</span>            if (snapshotTransmit != null &amp;&amp; snapshotTransmit.getSnapshotIndex() &lt; this.raft.lastAppliedIndex) {<a name="line.2743"></a>
<span class="sourceLineNo">2744</span>                if (this.log.isDebugEnabled())<a name="line.2744"></a>
<span class="sourceLineNo">2745</span>                    this.debug("aborting stale snapshot install for " + follower);<a name="line.2745"></a>
<span class="sourceLineNo">2746</span>                follower.cancelSnapshotTransmit();<a name="line.2746"></a>
<span class="sourceLineNo">2747</span>                follower.updateNow();<a name="line.2747"></a>
<span class="sourceLineNo">2748</span>            }<a name="line.2748"></a>
<span class="sourceLineNo">2749</span><a name="line.2749"></a>
<span class="sourceLineNo">2750</span>            // Is follower's queue empty? If not, hold off until then<a name="line.2750"></a>
<span class="sourceLineNo">2751</span>            if (this.raft.isTransmitting(follower.getAddress())) {<a name="line.2751"></a>
<span class="sourceLineNo">2752</span>                if (this.log.isTraceEnabled())<a name="line.2752"></a>
<span class="sourceLineNo">2753</span>                    this.trace("no update for \"" + peer + "\": output queue still not empty");<a name="line.2753"></a>
<span class="sourceLineNo">2754</span>                return;<a name="line.2754"></a>
<span class="sourceLineNo">2755</span>            }<a name="line.2755"></a>
<span class="sourceLineNo">2756</span><a name="line.2756"></a>
<span class="sourceLineNo">2757</span>            // Handle any in-progress snapshot install<a name="line.2757"></a>
<span class="sourceLineNo">2758</span>            if ((snapshotTransmit = follower.getSnapshotTransmit()) != null) {<a name="line.2758"></a>
<span class="sourceLineNo">2759</span><a name="line.2759"></a>
<span class="sourceLineNo">2760</span>                // Send the next chunk in transmission, if any<a name="line.2760"></a>
<span class="sourceLineNo">2761</span>                final long pairIndex = snapshotTransmit.getPairIndex();<a name="line.2761"></a>
<span class="sourceLineNo">2762</span>                final ByteBuffer chunk = snapshotTransmit.getNextChunk();<a name="line.2762"></a>
<span class="sourceLineNo">2763</span>                boolean synced = true;<a name="line.2763"></a>
<span class="sourceLineNo">2764</span>                if (chunk != null) {<a name="line.2764"></a>
<span class="sourceLineNo">2765</span><a name="line.2765"></a>
<span class="sourceLineNo">2766</span>                    // Send next chunk<a name="line.2766"></a>
<span class="sourceLineNo">2767</span>                    final InstallSnapshot msg = new InstallSnapshot(this.raft.clusterId, this.raft.identity, peer,<a name="line.2767"></a>
<span class="sourceLineNo">2768</span>                      this.raft.currentTerm, snapshotTransmit.getSnapshotTerm(), snapshotTransmit.getSnapshotIndex(), pairIndex,<a name="line.2768"></a>
<span class="sourceLineNo">2769</span>                      pairIndex == 0 ? snapshotTransmit.getSnapshotConfig() : null, !snapshotTransmit.hasMoreChunks(), chunk);<a name="line.2769"></a>
<span class="sourceLineNo">2770</span>                    if (this.raft.sendMessage(msg))<a name="line.2770"></a>
<span class="sourceLineNo">2771</span>                        return;<a name="line.2771"></a>
<span class="sourceLineNo">2772</span>                    if (this.log.isDebugEnabled())<a name="line.2772"></a>
<span class="sourceLineNo">2773</span>                        this.debug("canceling snapshot install for " + follower + " due to failure to send " + msg);<a name="line.2773"></a>
<span class="sourceLineNo">2774</span><a name="line.2774"></a>
<span class="sourceLineNo">2775</span>                    // Message failed -&gt; snapshot is fatally wounded, so cancel it<a name="line.2775"></a>
<span class="sourceLineNo">2776</span>                    synced = false;<a name="line.2776"></a>
<span class="sourceLineNo">2777</span>                }<a name="line.2777"></a>
<span class="sourceLineNo">2778</span>                if (synced)<a name="line.2778"></a>
<span class="sourceLineNo">2779</span>                    this.info("completed snapshot install for out-of-date " + follower);<a name="line.2779"></a>
<span class="sourceLineNo">2780</span><a name="line.2780"></a>
<span class="sourceLineNo">2781</span>                // Snapshot transmit is complete (or failed)<a name="line.2781"></a>
<span class="sourceLineNo">2782</span>                follower.cancelSnapshotTransmit();<a name="line.2782"></a>
<span class="sourceLineNo">2783</span><a name="line.2783"></a>
<span class="sourceLineNo">2784</span>                // Trigger an immediate regular update<a name="line.2784"></a>
<span class="sourceLineNo">2785</span>                follower.setNextIndex(snapshotTransmit.getSnapshotIndex() + 1);<a name="line.2785"></a>
<span class="sourceLineNo">2786</span>                follower.setSynced(synced);<a name="line.2786"></a>
<span class="sourceLineNo">2787</span>                follower.updateNow();<a name="line.2787"></a>
<span class="sourceLineNo">2788</span>                this.raft.requestService(new UpdateFollowerService(follower));<a name="line.2788"></a>
<span class="sourceLineNo">2789</span>                return;<a name="line.2789"></a>
<span class="sourceLineNo">2790</span>            }<a name="line.2790"></a>
<span class="sourceLineNo">2791</span><a name="line.2791"></a>
<span class="sourceLineNo">2792</span>            // Are we still waiting for the update timer to expire?<a name="line.2792"></a>
<span class="sourceLineNo">2793</span>            if (!follower.getUpdateTimer().pollForTimeout()) {<a name="line.2793"></a>
<span class="sourceLineNo">2794</span>                boolean waitForTimerToExpire = true;<a name="line.2794"></a>
<span class="sourceLineNo">2795</span><a name="line.2795"></a>
<span class="sourceLineNo">2796</span>                // Don't wait for the update timer to expire if:<a name="line.2796"></a>
<span class="sourceLineNo">2797</span>                //  (a) The follower is sync'd; AND<a name="line.2797"></a>
<span class="sourceLineNo">2798</span>                //      (y) We have a new log entry that the follower doesn't have; OR<a name="line.2798"></a>
<span class="sourceLineNo">2799</span>                //      (y) We have a new leaderCommit that the follower doesn't have<a name="line.2799"></a>
<span class="sourceLineNo">2800</span>                // The effect is that we will pipeline updates to synchronized followers.<a name="line.2800"></a>
<span class="sourceLineNo">2801</span>                if (follower.isSynced()<a name="line.2801"></a>
<span class="sourceLineNo">2802</span>                  &amp;&amp; (follower.getLeaderCommit() != this.raft.commitIndex<a name="line.2802"></a>
<span class="sourceLineNo">2803</span>                   || follower.getNextIndex() &lt;= this.raft.getLastLogIndex()))<a name="line.2803"></a>
<span class="sourceLineNo">2804</span>                    waitForTimerToExpire = false;<a name="line.2804"></a>
<span class="sourceLineNo">2805</span><a name="line.2805"></a>
<span class="sourceLineNo">2806</span>                // Wait for timer to expire<a name="line.2806"></a>
<span class="sourceLineNo">2807</span>                if (waitForTimerToExpire) {<a name="line.2807"></a>
<span class="sourceLineNo">2808</span>                    if (this.log.isTraceEnabled()) {<a name="line.2808"></a>
<span class="sourceLineNo">2809</span>                        this.trace("no update for \"" + follower.getIdentity() + "\": timer not expired yet, and follower is "<a name="line.2809"></a>
<span class="sourceLineNo">2810</span>                          + (follower.isSynced() ? "up to date" : "not synced"));<a name="line.2810"></a>
<span class="sourceLineNo">2811</span>                    }<a name="line.2811"></a>
<span class="sourceLineNo">2812</span>                    return;<a name="line.2812"></a>
<span class="sourceLineNo">2813</span>                }<a name="line.2813"></a>
<span class="sourceLineNo">2814</span>            }<a name="line.2814"></a>
<span class="sourceLineNo">2815</span><a name="line.2815"></a>
<span class="sourceLineNo">2816</span>            // Get index of the next log entry to send to follower<a name="line.2816"></a>
<span class="sourceLineNo">2817</span>            final long nextIndex = follower.getNextIndex();<a name="line.2817"></a>
<span class="sourceLineNo">2818</span><a name="line.2818"></a>
<span class="sourceLineNo">2819</span>            // If follower is too far behind, we must do a snapshot install<a name="line.2819"></a>
<span class="sourceLineNo">2820</span>            if (nextIndex &lt;= this.raft.lastAppliedIndex) {<a name="line.2820"></a>
<span class="sourceLineNo">2821</span>                final MostRecentView view = this.raft.new MostRecentView();<a name="line.2821"></a>
<span class="sourceLineNo">2822</span>                follower.setSnapshotTransmit(new SnapshotTransmit(this.raft.getLastLogTerm(), this.raft.getLastLogIndex(),<a name="line.2822"></a>
<span class="sourceLineNo">2823</span>                  this.raft.lastAppliedConfig, view.getSnapshot(), view.getView()));<a name="line.2823"></a>
<span class="sourceLineNo">2824</span>                this.info("started snapshot install for out-of-date " + follower);<a name="line.2824"></a>
<span class="sourceLineNo">2825</span>                this.raft.requestService(new UpdateFollowerService(follower));<a name="line.2825"></a>
<span class="sourceLineNo">2826</span>                return;<a name="line.2826"></a>
<span class="sourceLineNo">2827</span>            }<a name="line.2827"></a>
<span class="sourceLineNo">2828</span><a name="line.2828"></a>
<span class="sourceLineNo">2829</span>            // Restart update timer here (to avoid looping if an error occurs below)<a name="line.2829"></a>
<span class="sourceLineNo">2830</span>            follower.getUpdateTimer().timeoutAfter(this.raft.heartbeatTimeout);<a name="line.2830"></a>
<span class="sourceLineNo">2831</span><a name="line.2831"></a>
<span class="sourceLineNo">2832</span>            // Send actual data if follower is synced and there is a log entry to send; otherwise, just send a probe<a name="line.2832"></a>
<span class="sourceLineNo">2833</span>            final AppendRequest msg;<a name="line.2833"></a>
<span class="sourceLineNo">2834</span>            if (!follower.isSynced() || nextIndex &gt; this.raft.getLastLogIndex()) {<a name="line.2834"></a>
<span class="sourceLineNo">2835</span>                msg = new AppendRequest(this.raft.clusterId, this.raft.identity, peer, this.raft.currentTerm, new Timestamp(),<a name="line.2835"></a>
<span class="sourceLineNo">2836</span>                  this.leaseTimeout, this.raft.commitIndex, this.raft.getLogTermAtIndex(nextIndex - 1), nextIndex - 1);   // probe<a name="line.2836"></a>
<span class="sourceLineNo">2837</span>            } else {<a name="line.2837"></a>
<span class="sourceLineNo">2838</span><a name="line.2838"></a>
<span class="sourceLineNo">2839</span>                // Get log entry to send<a name="line.2839"></a>
<span class="sourceLineNo">2840</span>                final LogEntry logEntry = this.raft.getLogEntryAtIndex(nextIndex);<a name="line.2840"></a>
<span class="sourceLineNo">2841</span><a name="line.2841"></a>
<span class="sourceLineNo">2842</span>                // If the log entry correspond's to follower's transaction, don't send the data because follower already has it.<a name="line.2842"></a>
<span class="sourceLineNo">2843</span>                // But only do this optimization the first time, in case something goes wrong on the follower's end.<a name="line.2843"></a>
<span class="sourceLineNo">2844</span>                ByteBuffer mutationData = null;<a name="line.2844"></a>
<span class="sourceLineNo">2845</span>                if (!follower.getSkipDataLogEntries().remove(logEntry)) {<a name="line.2845"></a>
<span class="sourceLineNo">2846</span>                    try {<a name="line.2846"></a>
<span class="sourceLineNo">2847</span>                        mutationData = logEntry.getContent();<a name="line.2847"></a>
<span class="sourceLineNo">2848</span>                    } catch (IOException e) {<a name="line.2848"></a>
<span class="sourceLineNo">2849</span>                        this.error("error reading log file " + logEntry.getFile(), e);<a name="line.2849"></a>
<span class="sourceLineNo">2850</span>                        return;<a name="line.2850"></a>
<span class="sourceLineNo">2851</span>                    }<a name="line.2851"></a>
<span class="sourceLineNo">2852</span>                }<a name="line.2852"></a>
<span class="sourceLineNo">2853</span><a name="line.2853"></a>
<span class="sourceLineNo">2854</span>                // Create message<a name="line.2854"></a>
<span class="sourceLineNo">2855</span>                msg = new AppendRequest(this.raft.clusterId, this.raft.identity, peer, this.raft.currentTerm, new Timestamp(),<a name="line.2855"></a>
<span class="sourceLineNo">2856</span>                  this.leaseTimeout, this.raft.commitIndex, this.raft.getLogTermAtIndex(nextIndex - 1), nextIndex - 1,<a name="line.2856"></a>
<span class="sourceLineNo">2857</span>                  logEntry.getTerm(), mutationData);<a name="line.2857"></a>
<span class="sourceLineNo">2858</span>            }<a name="line.2858"></a>
<span class="sourceLineNo">2859</span><a name="line.2859"></a>
<span class="sourceLineNo">2860</span>            // Send update<a name="line.2860"></a>
<span class="sourceLineNo">2861</span>            final boolean sent = this.raft.sendMessage(msg);<a name="line.2861"></a>
<span class="sourceLineNo">2862</span><a name="line.2862"></a>
<span class="sourceLineNo">2863</span>            // Advance next index if a log entry was sent; we allow pipelining log entries when synchronized<a name="line.2863"></a>
<span class="sourceLineNo">2864</span>            if (sent &amp;&amp; !msg.isProbe()) {<a name="line.2864"></a>
<span class="sourceLineNo">2865</span>                assert follower.isSynced();<a name="line.2865"></a>
<span class="sourceLineNo">2866</span>                follower.setNextIndex(Math.min(follower.getNextIndex(), this.raft.getLastLogIndex()) + 1);<a name="line.2866"></a>
<span class="sourceLineNo">2867</span>            }<a name="line.2867"></a>
<span class="sourceLineNo">2868</span><a name="line.2868"></a>
<span class="sourceLineNo">2869</span>            // Update leaderCommit for follower<a name="line.2869"></a>
<span class="sourceLineNo">2870</span>            if (sent)<a name="line.2870"></a>
<span class="sourceLineNo">2871</span>                follower.setLeaderCommit(msg.getLeaderCommit());<a name="line.2871"></a>
<span class="sourceLineNo">2872</span>        }<a name="line.2872"></a>
<span class="sourceLineNo">2873</span><a name="line.2873"></a>
<span class="sourceLineNo">2874</span>        private void updateAllFollowersNow() {<a name="line.2874"></a>
<span class="sourceLineNo">2875</span>            for (Follower follower : this.followerMap.values())<a name="line.2875"></a>
<span class="sourceLineNo">2876</span>                follower.updateNow();<a name="line.2876"></a>
<span class="sourceLineNo">2877</span>        }<a name="line.2877"></a>
<span class="sourceLineNo">2878</span><a name="line.2878"></a>
<span class="sourceLineNo">2879</span>        private class UpdateFollowerService extends Service {<a name="line.2879"></a>
<span class="sourceLineNo">2880</span><a name="line.2880"></a>
<span class="sourceLineNo">2881</span>            private final Follower follower;<a name="line.2881"></a>
<span class="sourceLineNo">2882</span><a name="line.2882"></a>
<span class="sourceLineNo">2883</span>            UpdateFollowerService(Follower follower) {<a name="line.2883"></a>
<span class="sourceLineNo">2884</span>                super(LeaderRole.this, "update follower \"" + follower.getIdentity() + "\"");<a name="line.2884"></a>
<span class="sourceLineNo">2885</span>                assert follower != null;<a name="line.2885"></a>
<span class="sourceLineNo">2886</span>                this.follower = follower;<a name="line.2886"></a>
<span class="sourceLineNo">2887</span>            }<a name="line.2887"></a>
<span class="sourceLineNo">2888</span><a name="line.2888"></a>
<span class="sourceLineNo">2889</span>            @Override<a name="line.2889"></a>
<span class="sourceLineNo">2890</span>            public void run() {<a name="line.2890"></a>
<span class="sourceLineNo">2891</span>                LeaderRole.this.updateFollower(this.follower);<a name="line.2891"></a>
<span class="sourceLineNo">2892</span>            }<a name="line.2892"></a>
<span class="sourceLineNo">2893</span><a name="line.2893"></a>
<span class="sourceLineNo">2894</span>            @Override<a name="line.2894"></a>
<span class="sourceLineNo">2895</span>            public boolean equals(Object obj) {<a name="line.2895"></a>
<span class="sourceLineNo">2896</span>                if (obj == null || obj.getClass() != this.getClass())<a name="line.2896"></a>
<span class="sourceLineNo">2897</span>                    return false;<a name="line.2897"></a>
<span class="sourceLineNo">2898</span>                final UpdateFollowerService that = (UpdateFollowerService)obj;<a name="line.2898"></a>
<span class="sourceLineNo">2899</span>                return this.follower.equals(that.follower);<a name="line.2899"></a>
<span class="sourceLineNo">2900</span>            }<a name="line.2900"></a>
<span class="sourceLineNo">2901</span><a name="line.2901"></a>
<span class="sourceLineNo">2902</span>            @Override<a name="line.2902"></a>
<span class="sourceLineNo">2903</span>            public int hashCode() {<a name="line.2903"></a>
<span class="sourceLineNo">2904</span>                return this.follower.hashCode();<a name="line.2904"></a>
<span class="sourceLineNo">2905</span>            }<a name="line.2905"></a>
<span class="sourceLineNo">2906</span>        }<a name="line.2906"></a>
<span class="sourceLineNo">2907</span><a name="line.2907"></a>
<span class="sourceLineNo">2908</span>    // Transactions<a name="line.2908"></a>
<span class="sourceLineNo">2909</span><a name="line.2909"></a>
<span class="sourceLineNo">2910</span>        @Override<a name="line.2910"></a>
<span class="sourceLineNo">2911</span>        public void checkReadyTransaction(final RaftKVTransaction tx) {<a name="line.2911"></a>
<span class="sourceLineNo">2912</span><a name="line.2912"></a>
<span class="sourceLineNo">2913</span>            // Sanity check<a name="line.2913"></a>
<span class="sourceLineNo">2914</span>            assert Thread.holdsLock(this.raft);<a name="line.2914"></a>
<span class="sourceLineNo">2915</span>            assert tx.getState().equals(TxState.COMMIT_READY);<a name="line.2915"></a>
<span class="sourceLineNo">2916</span><a name="line.2916"></a>
<span class="sourceLineNo">2917</span>            // Check snapshot isolation<a name="line.2917"></a>
<span class="sourceLineNo">2918</span>            if (this.checkReadyTransactionReadOnlySnapshot(tx))<a name="line.2918"></a>
<span class="sourceLineNo">2919</span>                return;<a name="line.2919"></a>
<span class="sourceLineNo">2920</span><a name="line.2920"></a>
<span class="sourceLineNo">2921</span>            // Check for conflict<a name="line.2921"></a>
<span class="sourceLineNo">2922</span>            final String error = this.checkConflicts(tx.getBaseTerm(), tx.getBaseIndex(), tx.getMutableView().getReads());<a name="line.2922"></a>
<span class="sourceLineNo">2923</span>            if (error != null) {<a name="line.2923"></a>
<span class="sourceLineNo">2924</span>                if (this.log.isDebugEnabled())<a name="line.2924"></a>
<span class="sourceLineNo">2925</span>                    this.debug("local transaction " + tx + " failed due to conflict: " + error);<a name="line.2925"></a>
<span class="sourceLineNo">2926</span>                throw new RetryTransactionException(tx, error);<a name="line.2926"></a>
<span class="sourceLineNo">2927</span>            }<a name="line.2927"></a>
<span class="sourceLineNo">2928</span><a name="line.2928"></a>
<span class="sourceLineNo">2929</span>            // Handle read-only vs. read-write transaction<a name="line.2929"></a>
<span class="sourceLineNo">2930</span>            final Writes writes = tx.getMutableView().getWrites();<a name="line.2930"></a>
<span class="sourceLineNo">2931</span>            if (writes.isEmpty() &amp;&amp; tx.getConfigChange() == null) {<a name="line.2931"></a>
<span class="sourceLineNo">2932</span><a name="line.2932"></a>
<span class="sourceLineNo">2933</span>                // Set commit term and index from last log entry<a name="line.2933"></a>
<span class="sourceLineNo">2934</span>                tx.setCommitTerm(this.raft.getLastLogTerm());<a name="line.2934"></a>
<span class="sourceLineNo">2935</span>                tx.setCommitIndex(this.raft.getLastLogIndex());<a name="line.2935"></a>
<span class="sourceLineNo">2936</span>                if (this.log.isDebugEnabled()) {<a name="line.2936"></a>
<span class="sourceLineNo">2937</span>                    this.debug("commit is " + tx.getCommitTerm() + "/" + tx.getCommitIndex()<a name="line.2937"></a>
<span class="sourceLineNo">2938</span>                      + " for local read-only transaction " + tx);<a name="line.2938"></a>
<span class="sourceLineNo">2939</span>                }<a name="line.2939"></a>
<span class="sourceLineNo">2940</span><a name="line.2940"></a>
<span class="sourceLineNo">2941</span>                // We will be able to commit this transaction immediately<a name="line.2941"></a>
<span class="sourceLineNo">2942</span>                this.raft.requestService(new CheckWaitingTransactionService(tx));<a name="line.2942"></a>
<span class="sourceLineNo">2943</span>            } else {<a name="line.2943"></a>
<span class="sourceLineNo">2944</span><a name="line.2944"></a>
<span class="sourceLineNo">2945</span>                // Don't commit a new config change while there is any previous config change outstanding<a name="line.2945"></a>
<span class="sourceLineNo">2946</span>                if (tx.getConfigChange() != null &amp;&amp; this.isConfigChangeOutstanding())<a name="line.2946"></a>
<span class="sourceLineNo">2947</span>                    return;<a name="line.2947"></a>
<span class="sourceLineNo">2948</span><a name="line.2948"></a>
<span class="sourceLineNo">2949</span>                // Commit transaction as a new log entry<a name="line.2949"></a>
<span class="sourceLineNo">2950</span>                final LogEntry logEntry;<a name="line.2950"></a>
<span class="sourceLineNo">2951</span>                try {<a name="line.2951"></a>
<span class="sourceLineNo">2952</span>                    logEntry = this.applyNewLogEntry(this.raft.new NewLogEntry(tx));<a name="line.2952"></a>
<span class="sourceLineNo">2953</span>                } catch (Exception e) {<a name="line.2953"></a>
<span class="sourceLineNo">2954</span>                    throw new KVTransactionException(tx, "error attempting to persist transaction", e);<a name="line.2954"></a>
<span class="sourceLineNo">2955</span>                }<a name="line.2955"></a>
<span class="sourceLineNo">2956</span>                if (this.log.isDebugEnabled())<a name="line.2956"></a>
<span class="sourceLineNo">2957</span>                    this.debug("added log entry " + logEntry + " for local transaction " + tx);<a name="line.2957"></a>
<span class="sourceLineNo">2958</span><a name="line.2958"></a>
<span class="sourceLineNo">2959</span>                // Set commit term and index from new log entry<a name="line.2959"></a>
<span class="sourceLineNo">2960</span>                tx.setCommitTerm(logEntry.getTerm());<a name="line.2960"></a>
<span class="sourceLineNo">2961</span>                tx.setCommitIndex(logEntry.getIndex());<a name="line.2961"></a>
<span class="sourceLineNo">2962</span><a name="line.2962"></a>
<span class="sourceLineNo">2963</span>                // If there are no followers, we can commit this immediately<a name="line.2963"></a>
<span class="sourceLineNo">2964</span>                if (this.followerMap.isEmpty())<a name="line.2964"></a>
<span class="sourceLineNo">2965</span>                    this.raft.requestService(new CheckWaitingTransactionService(tx));<a name="line.2965"></a>
<span class="sourceLineNo">2966</span>            }<a name="line.2966"></a>
<span class="sourceLineNo">2967</span><a name="line.2967"></a>
<span class="sourceLineNo">2968</span>            // Update transaction state<a name="line.2968"></a>
<span class="sourceLineNo">2969</span>            tx.setState(TxState.COMMIT_WAITING);<a name="line.2969"></a>
<span class="sourceLineNo">2970</span>            tx.getMutableView().disableReadTracking();                                  // we no longer need this info<a name="line.2970"></a>
<span class="sourceLineNo">2971</span>        }<a name="line.2971"></a>
<span class="sourceLineNo">2972</span><a name="line.2972"></a>
<span class="sourceLineNo">2973</span>        @Override<a name="line.2973"></a>
<span class="sourceLineNo">2974</span>        public void cleanupForTransaction(RaftKVTransaction tx) {<a name="line.2974"></a>
<span class="sourceLineNo">2975</span>            // nothing to do<a name="line.2975"></a>
<span class="sourceLineNo">2976</span>        }<a name="line.2976"></a>
<span class="sourceLineNo">2977</span><a name="line.2977"></a>
<span class="sourceLineNo">2978</span>        private boolean isConfigChangeOutstanding() {<a name="line.2978"></a>
<span class="sourceLineNo">2979</span>            for (int i = (int)(this.raft.commitIndex - this.raft.lastAppliedIndex) + 1; i &lt; this.raft.raftLog.size(); i++) {<a name="line.2979"></a>
<span class="sourceLineNo">2980</span>                if (this.raft.raftLog.get(i).getConfigChange() != null)<a name="line.2980"></a>
<span class="sourceLineNo">2981</span>                    return true;<a name="line.2981"></a>
<span class="sourceLineNo">2982</span>            }<a name="line.2982"></a>
<span class="sourceLineNo">2983</span>            return false;<a name="line.2983"></a>
<span class="sourceLineNo">2984</span>        }<a name="line.2984"></a>
<span class="sourceLineNo">2985</span><a name="line.2985"></a>
<span class="sourceLineNo">2986</span>    // Message<a name="line.2986"></a>
<span class="sourceLineNo">2987</span><a name="line.2987"></a>
<span class="sourceLineNo">2988</span>        @Override<a name="line.2988"></a>
<span class="sourceLineNo">2989</span>        public void caseAppendRequest(AppendRequest msg) {<a name="line.2989"></a>
<span class="sourceLineNo">2990</span>            this.failDuplicateLeader(msg);<a name="line.2990"></a>
<span class="sourceLineNo">2991</span>        }<a name="line.2991"></a>
<span class="sourceLineNo">2992</span><a name="line.2992"></a>
<span class="sourceLineNo">2993</span>        @Override<a name="line.2993"></a>
<span class="sourceLineNo">2994</span>        public void caseAppendResponse(AppendResponse msg) {<a name="line.2994"></a>
<span class="sourceLineNo">2995</span><a name="line.2995"></a>
<span class="sourceLineNo">2996</span>            // Find follower<a name="line.2996"></a>
<span class="sourceLineNo">2997</span>            final Follower follower = this.findFollower(msg);<a name="line.2997"></a>
<span class="sourceLineNo">2998</span>            if (follower == null)<a name="line.2998"></a>
<span class="sourceLineNo">2999</span>                return;<a name="line.2999"></a>
<span class="sourceLineNo">3000</span><a name="line.3000"></a>
<span class="sourceLineNo">3001</span>            // Update follower's last rec'd leader timestamp<a name="line.3001"></a>
<span class="sourceLineNo">3002</span>            if (msg.getLeaderTimestamp().compareTo(follower.getLeaderTimestamp()) &gt; 0) {<a name="line.3002"></a>
<span class="sourceLineNo">3003</span>                follower.setLeaderTimestamp(msg.getLeaderTimestamp());<a name="line.3003"></a>
<span class="sourceLineNo">3004</span>                this.raft.requestService(this.updateLeaseTimeoutService);<a name="line.3004"></a>
<span class="sourceLineNo">3005</span>            }<a name="line.3005"></a>
<span class="sourceLineNo">3006</span><a name="line.3006"></a>
<span class="sourceLineNo">3007</span>            // Ignore if a snapshot install is in progress<a name="line.3007"></a>
<span class="sourceLineNo">3008</span>            if (follower.getSnapshotTransmit() != null) {<a name="line.3008"></a>
<span class="sourceLineNo">3009</span>                if (this.log.isTraceEnabled())<a name="line.3009"></a>
<span class="sourceLineNo">3010</span>                    this.trace("rec'd " + msg + " while sending snapshot install; ignoring");<a name="line.3010"></a>
<span class="sourceLineNo">3011</span>                return;<a name="line.3011"></a>
<span class="sourceLineNo">3012</span>            }<a name="line.3012"></a>
<span class="sourceLineNo">3013</span><a name="line.3013"></a>
<span class="sourceLineNo">3014</span>            // Flag indicating we might want to update follower when done<a name="line.3014"></a>
<span class="sourceLineNo">3015</span>            boolean updateFollowerAgain = false;<a name="line.3015"></a>
<span class="sourceLineNo">3016</span><a name="line.3016"></a>
<span class="sourceLineNo">3017</span>            // Update follower's match index<a name="line.3017"></a>
<span class="sourceLineNo">3018</span>            if (msg.getMatchIndex() &gt; follower.getMatchIndex()) {<a name="line.3018"></a>
<span class="sourceLineNo">3019</span>                follower.setMatchIndex(msg.getMatchIndex());<a name="line.3019"></a>
<span class="sourceLineNo">3020</span>                this.raft.requestService(this.updateLeaderCommitIndexService);<a name="line.3020"></a>
<span class="sourceLineNo">3021</span>                this.raft.requestService(this.applyCommittedLogEntriesService);<a name="line.3021"></a>
<span class="sourceLineNo">3022</span>                if (!this.raft.currentConfig.containsKey(follower.getIdentity()))<a name="line.3022"></a>
<span class="sourceLineNo">3023</span>                    this.raft.requestService(this.updateKnownFollowersService);<a name="line.3023"></a>
<span class="sourceLineNo">3024</span>            }<a name="line.3024"></a>
<span class="sourceLineNo">3025</span><a name="line.3025"></a>
<span class="sourceLineNo">3026</span>            // Check result and update follower's next index<a name="line.3026"></a>
<span class="sourceLineNo">3027</span>            final boolean wasSynced = follower.isSynced();<a name="line.3027"></a>
<span class="sourceLineNo">3028</span>            final long previousNextIndex = follower.getNextIndex();<a name="line.3028"></a>
<span class="sourceLineNo">3029</span>            if (!msg.isSuccess())<a name="line.3029"></a>
<span class="sourceLineNo">3030</span>                follower.setNextIndex(Math.max(follower.getNextIndex() - 1, 1));<a name="line.3030"></a>
<span class="sourceLineNo">3031</span>            follower.setSynced(msg.isSuccess());<a name="line.3031"></a>
<span class="sourceLineNo">3032</span>            if (follower.isSynced() != wasSynced) {<a name="line.3032"></a>
<span class="sourceLineNo">3033</span>                if (this.log.isDebugEnabled()) {<a name="line.3033"></a>
<span class="sourceLineNo">3034</span>                    this.debug("sync status of \"" + follower.getIdentity() + "\" changed -&gt; "<a name="line.3034"></a>
<span class="sourceLineNo">3035</span>                      + (!follower.isSynced() ? "not " : "") + "synced");<a name="line.3035"></a>
<span class="sourceLineNo">3036</span>                }<a name="line.3036"></a>
<span class="sourceLineNo">3037</span>                updateFollowerAgain = true;<a name="line.3037"></a>
<span class="sourceLineNo">3038</span>            }<a name="line.3038"></a>
<span class="sourceLineNo">3039</span><a name="line.3039"></a>
<span class="sourceLineNo">3040</span>            // Use follower's match index as a lower bound on follower's next index. This is needed because in this implementation,<a name="line.3040"></a>
<span class="sourceLineNo">3041</span>            // the application of leader log entries (to the state machine) may occur later than the application of follower log<a name="line.3041"></a>
<span class="sourceLineNo">3042</span>            // entries, so it's possible a follower's state machine's index can advance past the leader's. In that case we want<a name="line.3042"></a>
<span class="sourceLineNo">3043</span>            // to avoid the leader sending log entries that are prior to the follower's last committed index.<a name="line.3043"></a>
<span class="sourceLineNo">3044</span>            follower.setNextIndex(Math.max(follower.getNextIndex(), follower.getMatchIndex() + 1));<a name="line.3044"></a>
<span class="sourceLineNo">3045</span><a name="line.3045"></a>
<span class="sourceLineNo">3046</span>            // Use follower's last log index as an upper bound on follower's next index.<a name="line.3046"></a>
<span class="sourceLineNo">3047</span>            follower.setNextIndex(Math.min(msg.getLastLogIndex() + 1, follower.getNextIndex()));<a name="line.3047"></a>
<span class="sourceLineNo">3048</span><a name="line.3048"></a>
<span class="sourceLineNo">3049</span>            // Update follower again if next index has changed<a name="line.3049"></a>
<span class="sourceLineNo">3050</span>            updateFollowerAgain |= follower.getNextIndex() != previousNextIndex;<a name="line.3050"></a>
<span class="sourceLineNo">3051</span><a name="line.3051"></a>
<span class="sourceLineNo">3052</span>            // Debug<a name="line.3052"></a>
<span class="sourceLineNo">3053</span>            if (this.log.isTraceEnabled())<a name="line.3053"></a>
<span class="sourceLineNo">3054</span>                this.trace("updated follower: " + follower + ", update again = " + updateFollowerAgain);<a name="line.3054"></a>
<span class="sourceLineNo">3055</span><a name="line.3055"></a>
<span class="sourceLineNo">3056</span>            // Immediately update follower again (if appropriate)<a name="line.3056"></a>
<span class="sourceLineNo">3057</span>            if (updateFollowerAgain)<a name="line.3057"></a>
<span class="sourceLineNo">3058</span>                this.raft.requestService(new UpdateFollowerService(follower));<a name="line.3058"></a>
<span class="sourceLineNo">3059</span>        }<a name="line.3059"></a>
<span class="sourceLineNo">3060</span><a name="line.3060"></a>
<span class="sourceLineNo">3061</span>        @Override<a name="line.3061"></a>
<span class="sourceLineNo">3062</span>        public void caseCommitRequest(CommitRequest msg) {<a name="line.3062"></a>
<span class="sourceLineNo">3063</span><a name="line.3063"></a>
<span class="sourceLineNo">3064</span>            // Find follower<a name="line.3064"></a>
<span class="sourceLineNo">3065</span>            final Follower follower = this.findFollower(msg);<a name="line.3065"></a>
<span class="sourceLineNo">3066</span>            if (follower == null)<a name="line.3066"></a>
<span class="sourceLineNo">3067</span>                return;<a name="line.3067"></a>
<span class="sourceLineNo">3068</span><a name="line.3068"></a>
<span class="sourceLineNo">3069</span>            // Decode reads<a name="line.3069"></a>
<span class="sourceLineNo">3070</span>            final Reads reads;<a name="line.3070"></a>
<span class="sourceLineNo">3071</span>            try {<a name="line.3071"></a>
<span class="sourceLineNo">3072</span>                reads = Reads.deserialize(new ByteBufferInputStream(msg.getReadsData()));<a name="line.3072"></a>
<span class="sourceLineNo">3073</span>            } catch (Exception e) {<a name="line.3073"></a>
<span class="sourceLineNo">3074</span>                this.error("error decoding reads data in " + msg, e);<a name="line.3074"></a>
<span class="sourceLineNo">3075</span>                this.raft.sendMessage(new CommitResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3075"></a>
<span class="sourceLineNo">3076</span>                  this.raft.currentTerm, msg.getTxId(), "error decoding reads data: " + e));<a name="line.3076"></a>
<span class="sourceLineNo">3077</span>                return;<a name="line.3077"></a>
<span class="sourceLineNo">3078</span>            }<a name="line.3078"></a>
<span class="sourceLineNo">3079</span><a name="line.3079"></a>
<span class="sourceLineNo">3080</span>            // Check for conflict<a name="line.3080"></a>
<span class="sourceLineNo">3081</span>            final String conflictMsg = this.checkConflicts(msg.getBaseTerm(), msg.getBaseIndex(), reads);<a name="line.3081"></a>
<span class="sourceLineNo">3082</span>            if (conflictMsg != null) {<a name="line.3082"></a>
<span class="sourceLineNo">3083</span>                if (this.log.isDebugEnabled())<a name="line.3083"></a>
<span class="sourceLineNo">3084</span>                    this.debug("commit request " + msg + " failed due to conflict: " + conflictMsg);<a name="line.3084"></a>
<span class="sourceLineNo">3085</span>                this.raft.sendMessage(new CommitResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3085"></a>
<span class="sourceLineNo">3086</span>                  this.raft.currentTerm, msg.getTxId(), conflictMsg));<a name="line.3086"></a>
<span class="sourceLineNo">3087</span>                return;<a name="line.3087"></a>
<span class="sourceLineNo">3088</span>            }<a name="line.3088"></a>
<span class="sourceLineNo">3089</span><a name="line.3089"></a>
<span class="sourceLineNo">3090</span>            // Handle read-only vs. read-write transaction<a name="line.3090"></a>
<span class="sourceLineNo">3091</span>            if (msg.isReadOnly()) {<a name="line.3091"></a>
<span class="sourceLineNo">3092</span><a name="line.3092"></a>
<span class="sourceLineNo">3093</span>                // Get current time<a name="line.3093"></a>
<span class="sourceLineNo">3094</span>                final Timestamp minimumLeaseTimeout = new Timestamp();<a name="line.3094"></a>
<span class="sourceLineNo">3095</span><a name="line.3095"></a>
<span class="sourceLineNo">3096</span>                // If no other leader could have been elected yet, the transaction may be committed immediately<a name="line.3096"></a>
<span class="sourceLineNo">3097</span>                final CommitResponse response;<a name="line.3097"></a>
<span class="sourceLineNo">3098</span>                if (this.leaseTimeout.compareTo(minimumLeaseTimeout) &gt; 0) {<a name="line.3098"></a>
<span class="sourceLineNo">3099</span><a name="line.3099"></a>
<span class="sourceLineNo">3100</span>                    // Follower may commit as soon as it sees that the transaction's base log entry has been committed.<a name="line.3100"></a>
<span class="sourceLineNo">3101</span>                    // Note, we don't need to wait for subsequent log entries to be committed, because even if they<a name="line.3101"></a>
<span class="sourceLineNo">3102</span>                    // never are, whatever log entries replace them will necessarily get created sometime after now.<a name="line.3102"></a>
<span class="sourceLineNo">3103</span>                    response = new CommitResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3103"></a>
<span class="sourceLineNo">3104</span>                      this.raft.currentTerm, msg.getTxId(), msg.getBaseTerm(), msg.getBaseIndex());<a name="line.3104"></a>
<span class="sourceLineNo">3105</span>                } else {<a name="line.3105"></a>
<span class="sourceLineNo">3106</span><a name="line.3106"></a>
<span class="sourceLineNo">3107</span>                    // Remember that this follower is now going to be waiting for this particular leaseTimeout<a name="line.3107"></a>
<span class="sourceLineNo">3108</span>                    follower.getCommitLeaseTimeouts().add(minimumLeaseTimeout);<a name="line.3108"></a>
<span class="sourceLineNo">3109</span><a name="line.3109"></a>
<span class="sourceLineNo">3110</span>                    // Send immediate probes to all followers in an attempt to increase our leaseTimeout<a name="line.3110"></a>
<span class="sourceLineNo">3111</span>                    this.updateAllFollowersNow();<a name="line.3111"></a>
<span class="sourceLineNo">3112</span><a name="line.3112"></a>
<span class="sourceLineNo">3113</span>                    // Build response<a name="line.3113"></a>
<span class="sourceLineNo">3114</span>                    response = new CommitResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(), this.raft.currentTerm,<a name="line.3114"></a>
<span class="sourceLineNo">3115</span>                      msg.getTxId(), this.raft.getLastLogTerm(), this.raft.getLastLogIndex(), minimumLeaseTimeout);<a name="line.3115"></a>
<span class="sourceLineNo">3116</span>                }<a name="line.3116"></a>
<span class="sourceLineNo">3117</span><a name="line.3117"></a>
<span class="sourceLineNo">3118</span>                // Send response<a name="line.3118"></a>
<span class="sourceLineNo">3119</span>                this.raft.sendMessage(response);<a name="line.3119"></a>
<span class="sourceLineNo">3120</span>            } else {<a name="line.3120"></a>
<span class="sourceLineNo">3121</span><a name="line.3121"></a>
<span class="sourceLineNo">3122</span>                // Commit mutations as a new log entry<a name="line.3122"></a>
<span class="sourceLineNo">3123</span>                final LogEntry logEntry;<a name="line.3123"></a>
<span class="sourceLineNo">3124</span>                try {<a name="line.3124"></a>
<span class="sourceLineNo">3125</span>                    logEntry = this.applyNewLogEntry(this.raft.new NewLogEntry(msg.getMutationData()));<a name="line.3125"></a>
<span class="sourceLineNo">3126</span>                } catch (Exception e) {<a name="line.3126"></a>
<span class="sourceLineNo">3127</span>                    this.error("error appending new log entry for " + msg, e);<a name="line.3127"></a>
<span class="sourceLineNo">3128</span>                    this.raft.sendMessage(new CommitResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3128"></a>
<span class="sourceLineNo">3129</span>                      this.raft.currentTerm, msg.getTxId(), e.getMessage() != null ? e.getMessage() : "" + e));<a name="line.3129"></a>
<span class="sourceLineNo">3130</span>                    return;<a name="line.3130"></a>
<span class="sourceLineNo">3131</span>                }<a name="line.3131"></a>
<span class="sourceLineNo">3132</span>                if (this.log.isDebugEnabled())<a name="line.3132"></a>
<span class="sourceLineNo">3133</span>                    this.debug("added log entry " + logEntry + " for remote " + msg);<a name="line.3133"></a>
<span class="sourceLineNo">3134</span><a name="line.3134"></a>
<span class="sourceLineNo">3135</span>                // Follower transaction data optimization<a name="line.3135"></a>
<span class="sourceLineNo">3136</span>                follower.getSkipDataLogEntries().add(logEntry);<a name="line.3136"></a>
<span class="sourceLineNo">3137</span><a name="line.3137"></a>
<span class="sourceLineNo">3138</span>                // Send response<a name="line.3138"></a>
<span class="sourceLineNo">3139</span>                this.raft.sendMessage(new CommitResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3139"></a>
<span class="sourceLineNo">3140</span>                  this.raft.currentTerm, msg.getTxId(), this.raft.getLastLogTerm(), this.raft.getLastLogIndex()));<a name="line.3140"></a>
<span class="sourceLineNo">3141</span>            }<a name="line.3141"></a>
<span class="sourceLineNo">3142</span>        }<a name="line.3142"></a>
<span class="sourceLineNo">3143</span><a name="line.3143"></a>
<span class="sourceLineNo">3144</span>        @Override<a name="line.3144"></a>
<span class="sourceLineNo">3145</span>        public void caseCommitResponse(CommitResponse msg) {<a name="line.3145"></a>
<span class="sourceLineNo">3146</span>            this.failDuplicateLeader(msg);<a name="line.3146"></a>
<span class="sourceLineNo">3147</span>        }<a name="line.3147"></a>
<span class="sourceLineNo">3148</span><a name="line.3148"></a>
<span class="sourceLineNo">3149</span>        @Override<a name="line.3149"></a>
<span class="sourceLineNo">3150</span>        public void caseInstallSnapshot(InstallSnapshot msg) {<a name="line.3150"></a>
<span class="sourceLineNo">3151</span>            this.failDuplicateLeader(msg);<a name="line.3151"></a>
<span class="sourceLineNo">3152</span>        }<a name="line.3152"></a>
<span class="sourceLineNo">3153</span><a name="line.3153"></a>
<span class="sourceLineNo">3154</span>        @Override<a name="line.3154"></a>
<span class="sourceLineNo">3155</span>        public void caseRequestVote(RequestVote msg) {<a name="line.3155"></a>
<span class="sourceLineNo">3156</span><a name="line.3156"></a>
<span class="sourceLineNo">3157</span>            // Too late dude, I already won the election<a name="line.3157"></a>
<span class="sourceLineNo">3158</span>            if (this.log.isDebugEnabled())<a name="line.3158"></a>
<span class="sourceLineNo">3159</span>                this.debug("ignoring " + msg + " rec'd while in " + this);<a name="line.3159"></a>
<span class="sourceLineNo">3160</span>        }<a name="line.3160"></a>
<span class="sourceLineNo">3161</span><a name="line.3161"></a>
<span class="sourceLineNo">3162</span>        @Override<a name="line.3162"></a>
<span class="sourceLineNo">3163</span>        public void caseGrantVote(GrantVote msg) {<a name="line.3163"></a>
<span class="sourceLineNo">3164</span><a name="line.3164"></a>
<span class="sourceLineNo">3165</span>            // Thanks and all, but I already won the election<a name="line.3165"></a>
<span class="sourceLineNo">3166</span>            if (this.log.isDebugEnabled())<a name="line.3166"></a>
<span class="sourceLineNo">3167</span>                this.debug("ignoring " + msg + " rec'd while in " + this);<a name="line.3167"></a>
<span class="sourceLineNo">3168</span>        }<a name="line.3168"></a>
<span class="sourceLineNo">3169</span><a name="line.3169"></a>
<span class="sourceLineNo">3170</span>        private void failDuplicateLeader(Message msg) {<a name="line.3170"></a>
<span class="sourceLineNo">3171</span><a name="line.3171"></a>
<span class="sourceLineNo">3172</span>            // This should never happen - same term but two different leaders<a name="line.3172"></a>
<span class="sourceLineNo">3173</span>            final boolean defer = this.raft.identity.compareTo(msg.getSenderId()) &lt;= 0;<a name="line.3173"></a>
<span class="sourceLineNo">3174</span>            this.error("detected a duplicate leader in " + msg + " - should never happen; possible inconsistent cluster"<a name="line.3174"></a>
<span class="sourceLineNo">3175</span>              + " configuration on " + msg.getSenderId() + " (mine: " + this.raft.currentConfig + "); "<a name="line.3175"></a>
<span class="sourceLineNo">3176</span>              + (defer ? "reverting to follower" : "ignoring"));<a name="line.3176"></a>
<span class="sourceLineNo">3177</span>            if (defer)<a name="line.3177"></a>
<span class="sourceLineNo">3178</span>                this.raft.changeRole(new FollowerRole(this.raft, msg.getSenderId(), this.raft.returnAddress));<a name="line.3178"></a>
<span class="sourceLineNo">3179</span>        }<a name="line.3179"></a>
<span class="sourceLineNo">3180</span><a name="line.3180"></a>
<span class="sourceLineNo">3181</span>    // Object<a name="line.3181"></a>
<span class="sourceLineNo">3182</span><a name="line.3182"></a>
<span class="sourceLineNo">3183</span>        @Override<a name="line.3183"></a>
<span class="sourceLineNo">3184</span>        public String toString() {<a name="line.3184"></a>
<span class="sourceLineNo">3185</span>            return this.toStringPrefix()<a name="line.3185"></a>
<span class="sourceLineNo">3186</span>              + ",followerMap=" + this.followerMap<a name="line.3186"></a>
<span class="sourceLineNo">3187</span>              + "]";<a name="line.3187"></a>
<span class="sourceLineNo">3188</span>        }<a name="line.3188"></a>
<span class="sourceLineNo">3189</span><a name="line.3189"></a>
<span class="sourceLineNo">3190</span>    // Debug<a name="line.3190"></a>
<span class="sourceLineNo">3191</span><a name="line.3191"></a>
<span class="sourceLineNo">3192</span>        @Override<a name="line.3192"></a>
<span class="sourceLineNo">3193</span>        boolean checkState() {<a name="line.3193"></a>
<span class="sourceLineNo">3194</span>            if (!super.checkState())<a name="line.3194"></a>
<span class="sourceLineNo">3195</span>                return false;<a name="line.3195"></a>
<span class="sourceLineNo">3196</span>            for (Follower follower : this.followerMap.values()) {<a name="line.3196"></a>
<span class="sourceLineNo">3197</span>                assert follower.getNextIndex() &lt;= this.raft.getLastLogIndex() + 1;<a name="line.3197"></a>
<span class="sourceLineNo">3198</span>                assert follower.getMatchIndex() &lt;= this.raft.getLastLogIndex() + 1;<a name="line.3198"></a>
<span class="sourceLineNo">3199</span>                assert follower.getLeaderCommit() &lt;= this.raft.commitIndex;<a name="line.3199"></a>
<span class="sourceLineNo">3200</span>                assert follower.getUpdateTimer().isRunning() || follower.getSnapshotTransmit() != null;<a name="line.3200"></a>
<span class="sourceLineNo">3201</span>            }<a name="line.3201"></a>
<span class="sourceLineNo">3202</span>            long index = this.raft.lastAppliedIndex - this.appliedLogEntries.size() + 1;<a name="line.3202"></a>
<span class="sourceLineNo">3203</span>            for (LogEntry logEntry : Iterables.concat(this.appliedLogEntries, this.raft.raftLog))<a name="line.3203"></a>
<span class="sourceLineNo">3204</span>                assert logEntry.getIndex() == index++;<a name="line.3204"></a>
<span class="sourceLineNo">3205</span>            return true;<a name="line.3205"></a>
<span class="sourceLineNo">3206</span>        }<a name="line.3206"></a>
<span class="sourceLineNo">3207</span><a name="line.3207"></a>
<span class="sourceLineNo">3208</span>    // Internal methods<a name="line.3208"></a>
<span class="sourceLineNo">3209</span><a name="line.3209"></a>
<span class="sourceLineNo">3210</span>        /**<a name="line.3210"></a>
<span class="sourceLineNo">3211</span>         * Find the index of the most recent unapplied log entry having an associated config change matching the given predicate.<a name="line.3211"></a>
<span class="sourceLineNo">3212</span>         *<a name="line.3212"></a>
<span class="sourceLineNo">3213</span>         * @return most recent matching log entry, or zero if none found<a name="line.3213"></a>
<span class="sourceLineNo">3214</span>         */<a name="line.3214"></a>
<span class="sourceLineNo">3215</span>        private long findMostRecentConfigChangeMatching(Predicate&lt;String[]&gt; predicate) {<a name="line.3215"></a>
<span class="sourceLineNo">3216</span>            for (long index = this.raft.getLastLogIndex(); index &gt; this.raft.lastAppliedIndex; index--) {<a name="line.3216"></a>
<span class="sourceLineNo">3217</span>                final String[] configChange = this.raft.getLogEntryAtIndex(index).getConfigChange();<a name="line.3217"></a>
<span class="sourceLineNo">3218</span>                if (configChange == null &amp;&amp; predicate.apply(configChange))<a name="line.3218"></a>
<span class="sourceLineNo">3219</span>                    return index;<a name="line.3219"></a>
<span class="sourceLineNo">3220</span>            }<a name="line.3220"></a>
<span class="sourceLineNo">3221</span>            return 0;<a name="line.3221"></a>
<span class="sourceLineNo">3222</span>        }<a name="line.3222"></a>
<span class="sourceLineNo">3223</span><a name="line.3223"></a>
<span class="sourceLineNo">3224</span>        // Apply a new log entry to the Raft log<a name="line.3224"></a>
<span class="sourceLineNo">3225</span>        private LogEntry applyNewLogEntry(NewLogEntry newLogEntry) throws Exception {<a name="line.3225"></a>
<span class="sourceLineNo">3226</span><a name="line.3226"></a>
<span class="sourceLineNo">3227</span>            // Do a couple of extra checks if a config change is included<a name="line.3227"></a>
<span class="sourceLineNo">3228</span>            final String[] configChange = newLogEntry.getData().getConfigChange();<a name="line.3228"></a>
<span class="sourceLineNo">3229</span>            if (configChange != null) {<a name="line.3229"></a>
<span class="sourceLineNo">3230</span><a name="line.3230"></a>
<span class="sourceLineNo">3231</span>                // Disallow a config change while there is a previous uncommitted config change<a name="line.3231"></a>
<span class="sourceLineNo">3232</span>                if (this.isConfigChangeOutstanding()) {<a name="line.3232"></a>
<span class="sourceLineNo">3233</span>                    newLogEntry.cancel();<a name="line.3233"></a>
<span class="sourceLineNo">3234</span>                    throw new IllegalStateException("uncommitted config change outstanding");<a name="line.3234"></a>
<span class="sourceLineNo">3235</span>                }<a name="line.3235"></a>
<span class="sourceLineNo">3236</span><a name="line.3236"></a>
<span class="sourceLineNo">3237</span>                // Disallow a configuration change that removes the last node in a cluster<a name="line.3237"></a>
<span class="sourceLineNo">3238</span>                if (this.raft.currentConfig.size() == 1 &amp;&amp; configChange[1] == null) {<a name="line.3238"></a>
<span class="sourceLineNo">3239</span>                    final String lastNode = this.raft.currentConfig.keySet().iterator().next();<a name="line.3239"></a>
<span class="sourceLineNo">3240</span>                    if (configChange[0].equals(lastNode)) {<a name="line.3240"></a>
<span class="sourceLineNo">3241</span>                        newLogEntry.cancel();<a name="line.3241"></a>
<span class="sourceLineNo">3242</span>                        throw new IllegalArgumentException("can't remove the last node in a cluster (\"" + lastNode + "\")");<a name="line.3242"></a>
<span class="sourceLineNo">3243</span>                    }<a name="line.3243"></a>
<span class="sourceLineNo">3244</span>                }<a name="line.3244"></a>
<span class="sourceLineNo">3245</span>            }<a name="line.3245"></a>
<span class="sourceLineNo">3246</span><a name="line.3246"></a>
<span class="sourceLineNo">3247</span>            // Append a new entry to the Raft log<a name="line.3247"></a>
<span class="sourceLineNo">3248</span>            final LogEntry logEntry;<a name="line.3248"></a>
<span class="sourceLineNo">3249</span>            boolean success = false;<a name="line.3249"></a>
<span class="sourceLineNo">3250</span>            try {<a name="line.3250"></a>
<span class="sourceLineNo">3251</span>                logEntry = this.raft.appendLogEntry(this.raft.currentTerm, newLogEntry);<a name="line.3251"></a>
<span class="sourceLineNo">3252</span>                success = true;<a name="line.3252"></a>
<span class="sourceLineNo">3253</span>            } finally {<a name="line.3253"></a>
<span class="sourceLineNo">3254</span>                if (!success)<a name="line.3254"></a>
<span class="sourceLineNo">3255</span>                    newLogEntry.cancel();<a name="line.3255"></a>
<span class="sourceLineNo">3256</span>            }<a name="line.3256"></a>
<span class="sourceLineNo">3257</span><a name="line.3257"></a>
<span class="sourceLineNo">3258</span>            // Update follower list if configuration changed<a name="line.3258"></a>
<span class="sourceLineNo">3259</span>            if (configChange != null)<a name="line.3259"></a>
<span class="sourceLineNo">3260</span>                this.raft.requestService(this.updateKnownFollowersService);<a name="line.3260"></a>
<span class="sourceLineNo">3261</span><a name="line.3261"></a>
<span class="sourceLineNo">3262</span>            // Prune applied log entries (check memory limit)<a name="line.3262"></a>
<span class="sourceLineNo">3263</span>            this.raft.requestService(this.pruneAppliedLogEntriesService);<a name="line.3263"></a>
<span class="sourceLineNo">3264</span><a name="line.3264"></a>
<span class="sourceLineNo">3265</span>            // Update commit index (this is only needed if config has changed, or in the single node case)<a name="line.3265"></a>
<span class="sourceLineNo">3266</span>            if (configChange != null || this.followerMap.isEmpty())<a name="line.3266"></a>
<span class="sourceLineNo">3267</span>                this.raft.requestService(this.updateLeaderCommitIndexService);<a name="line.3267"></a>
<span class="sourceLineNo">3268</span><a name="line.3268"></a>
<span class="sourceLineNo">3269</span>            // Immediately update all up-to-date followers<a name="line.3269"></a>
<span class="sourceLineNo">3270</span>            this.updateAllFollowersNow();<a name="line.3270"></a>
<span class="sourceLineNo">3271</span><a name="line.3271"></a>
<span class="sourceLineNo">3272</span>            // Done<a name="line.3272"></a>
<span class="sourceLineNo">3273</span>            return logEntry;<a name="line.3273"></a>
<span class="sourceLineNo">3274</span>        }<a name="line.3274"></a>
<span class="sourceLineNo">3275</span><a name="line.3275"></a>
<span class="sourceLineNo">3276</span>        /**<a name="line.3276"></a>
<span class="sourceLineNo">3277</span>         * Check whether a proposed transaction can commit without any MVCC conflict.<a name="line.3277"></a>
<span class="sourceLineNo">3278</span>         *<a name="line.3278"></a>
<span class="sourceLineNo">3279</span>         * @param file file containing serialized copy of {@link writes} (content must already be fsync()'d to disk!)<a name="line.3279"></a>
<span class="sourceLineNo">3280</span>         * @param baseTerm the term of the log entry on which the transaction is based<a name="line.3280"></a>
<span class="sourceLineNo">3281</span>         * @param baseIndex the index of the log entry on which the transaction is based<a name="line.3281"></a>
<span class="sourceLineNo">3282</span>         * @param reads reads performed by the transaction<a name="line.3282"></a>
<span class="sourceLineNo">3283</span>         * @param writes writes performed by the transaction<a name="line.3283"></a>
<span class="sourceLineNo">3284</span>         * @return error message on failure, null for success<a name="line.3284"></a>
<span class="sourceLineNo">3285</span>         */<a name="line.3285"></a>
<span class="sourceLineNo">3286</span>        public String checkConflicts(long baseTerm, long baseIndex, Reads reads) {<a name="line.3286"></a>
<span class="sourceLineNo">3287</span><a name="line.3287"></a>
<span class="sourceLineNo">3288</span>            // Validate the index of the log entry on which the transaction is based<a name="line.3288"></a>
<span class="sourceLineNo">3289</span>            final long minIndex = this.raft.lastAppliedIndex;<a name="line.3289"></a>
<span class="sourceLineNo">3290</span>            final long maxIndex = this.raft.getLastLogIndex();<a name="line.3290"></a>
<span class="sourceLineNo">3291</span>            if (baseIndex &lt; minIndex)<a name="line.3291"></a>
<span class="sourceLineNo">3292</span>                return "transaction is too old: snapshot index " + baseIndex + " &lt; current state machine index " + minIndex;<a name="line.3292"></a>
<span class="sourceLineNo">3293</span>            if (baseIndex &gt; maxIndex)<a name="line.3293"></a>
<span class="sourceLineNo">3294</span>                return "transaction is too new: snapshot index " + baseIndex + " &gt; most recent log index " + maxIndex;<a name="line.3294"></a>
<span class="sourceLineNo">3295</span><a name="line.3295"></a>
<span class="sourceLineNo">3296</span>            // Validate the term of the log entry on which the transaction is based<a name="line.3296"></a>
<span class="sourceLineNo">3297</span>            if (baseTerm != this.raft.getLogTermAtIndex(baseIndex)) {<a name="line.3297"></a>
<span class="sourceLineNo">3298</span>                return "transaction is based on an overwritten log entry with index "<a name="line.3298"></a>
<span class="sourceLineNo">3299</span>                  + baseIndex + " and term " + baseTerm + " != " + this.raft.getLogTermAtIndex(baseIndex);<a name="line.3299"></a>
<span class="sourceLineNo">3300</span>            }<a name="line.3300"></a>
<span class="sourceLineNo">3301</span><a name="line.3301"></a>
<span class="sourceLineNo">3302</span>            // Check for conflicts from intervening commits<a name="line.3302"></a>
<span class="sourceLineNo">3303</span>            for (long index = baseIndex + 1; index &lt;= maxIndex; index++) {<a name="line.3303"></a>
<span class="sourceLineNo">3304</span>                final LogEntry logEntry = this.raft.getLogEntryAtIndex(index);<a name="line.3304"></a>
<span class="sourceLineNo">3305</span>                if (reads.isConflict(logEntry.getWrites())) {<a name="line.3305"></a>
<span class="sourceLineNo">3306</span>                    return "writes of committed transaction at index " + index<a name="line.3306"></a>
<span class="sourceLineNo">3307</span>                      + " conflict with transaction reads from transaction base index " + baseIndex;<a name="line.3307"></a>
<span class="sourceLineNo">3308</span>                }<a name="line.3308"></a>
<span class="sourceLineNo">3309</span>            }<a name="line.3309"></a>
<span class="sourceLineNo">3310</span><a name="line.3310"></a>
<span class="sourceLineNo">3311</span>            // No conflict<a name="line.3311"></a>
<span class="sourceLineNo">3312</span>            return null;<a name="line.3312"></a>
<span class="sourceLineNo">3313</span>        }<a name="line.3313"></a>
<span class="sourceLineNo">3314</span><a name="line.3314"></a>
<span class="sourceLineNo">3315</span>        private Follower findFollower(Message msg) {<a name="line.3315"></a>
<span class="sourceLineNo">3316</span>            final Follower follower = this.followerMap.get(msg.getSenderId());<a name="line.3316"></a>
<span class="sourceLineNo">3317</span>            if (follower == null)<a name="line.3317"></a>
<span class="sourceLineNo">3318</span>                this.warn("rec'd " + msg + " from unknown follower \"" + msg.getSenderId() + "\", ignoring");<a name="line.3318"></a>
<span class="sourceLineNo">3319</span>            return follower;<a name="line.3319"></a>
<span class="sourceLineNo">3320</span>        }<a name="line.3320"></a>
<span class="sourceLineNo">3321</span>    }<a name="line.3321"></a>
<span class="sourceLineNo">3322</span><a name="line.3322"></a>
<span class="sourceLineNo">3323</span>    /**<a name="line.3323"></a>
<span class="sourceLineNo">3324</span>     * Support superclass for {@link FollowerRole} and {@link CandidateRole}, which both have an election timer.<a name="line.3324"></a>
<span class="sourceLineNo">3325</span>     */<a name="line.3325"></a>
<span class="sourceLineNo">3326</span>    private abstract static class NonLeaderRole extends Role {<a name="line.3326"></a>
<span class="sourceLineNo">3327</span><a name="line.3327"></a>
<span class="sourceLineNo">3328</span>        protected final Timer electionTimer = this.raft.new Timer("election timer", new Service(this, "election timeout") {<a name="line.3328"></a>
<span class="sourceLineNo">3329</span>            @Override<a name="line.3329"></a>
<span class="sourceLineNo">3330</span>            public void run() {<a name="line.3330"></a>
<span class="sourceLineNo">3331</span>                NonLeaderRole.this.checkElectionTimeout();<a name="line.3331"></a>
<span class="sourceLineNo">3332</span>            }<a name="line.3332"></a>
<span class="sourceLineNo">3333</span>        });<a name="line.3333"></a>
<span class="sourceLineNo">3334</span>        private final boolean startElectionTimer;<a name="line.3334"></a>
<span class="sourceLineNo">3335</span><a name="line.3335"></a>
<span class="sourceLineNo">3336</span>    // Constructors<a name="line.3336"></a>
<span class="sourceLineNo">3337</span><a name="line.3337"></a>
<span class="sourceLineNo">3338</span>        protected NonLeaderRole(RaftKVDatabase raft, boolean startElectionTimer) {<a name="line.3338"></a>
<span class="sourceLineNo">3339</span>            super(raft);<a name="line.3339"></a>
<span class="sourceLineNo">3340</span>            this.startElectionTimer = startElectionTimer;<a name="line.3340"></a>
<span class="sourceLineNo">3341</span>        }<a name="line.3341"></a>
<span class="sourceLineNo">3342</span><a name="line.3342"></a>
<span class="sourceLineNo">3343</span>    // Lifecycle<a name="line.3343"></a>
<span class="sourceLineNo">3344</span><a name="line.3344"></a>
<span class="sourceLineNo">3345</span>        @Override<a name="line.3345"></a>
<span class="sourceLineNo">3346</span>        public void setup() {<a name="line.3346"></a>
<span class="sourceLineNo">3347</span>            super.setup();<a name="line.3347"></a>
<span class="sourceLineNo">3348</span>            if (this.startElectionTimer)<a name="line.3348"></a>
<span class="sourceLineNo">3349</span>                this.restartElectionTimer();<a name="line.3349"></a>
<span class="sourceLineNo">3350</span>        }<a name="line.3350"></a>
<span class="sourceLineNo">3351</span><a name="line.3351"></a>
<span class="sourceLineNo">3352</span>        @Override<a name="line.3352"></a>
<span class="sourceLineNo">3353</span>        public void shutdown() {<a name="line.3353"></a>
<span class="sourceLineNo">3354</span>            super.shutdown();<a name="line.3354"></a>
<span class="sourceLineNo">3355</span>            this.electionTimer.cancel();<a name="line.3355"></a>
<span class="sourceLineNo">3356</span>        }<a name="line.3356"></a>
<span class="sourceLineNo">3357</span><a name="line.3357"></a>
<span class="sourceLineNo">3358</span>    // Service<a name="line.3358"></a>
<span class="sourceLineNo">3359</span><a name="line.3359"></a>
<span class="sourceLineNo">3360</span>        // Check for an election timeout<a name="line.3360"></a>
<span class="sourceLineNo">3361</span>        private void checkElectionTimeout() {<a name="line.3361"></a>
<span class="sourceLineNo">3362</span>            if (this.electionTimer.pollForTimeout()) {<a name="line.3362"></a>
<span class="sourceLineNo">3363</span>                this.info("election timeout while in " + this);<a name="line.3363"></a>
<span class="sourceLineNo">3364</span>                this.raft.changeRole(new CandidateRole(this.raft));<a name="line.3364"></a>
<span class="sourceLineNo">3365</span>            }<a name="line.3365"></a>
<span class="sourceLineNo">3366</span>        }<a name="line.3366"></a>
<span class="sourceLineNo">3367</span><a name="line.3367"></a>
<span class="sourceLineNo">3368</span>        protected void restartElectionTimer() {<a name="line.3368"></a>
<span class="sourceLineNo">3369</span><a name="line.3369"></a>
<span class="sourceLineNo">3370</span>            // Sanity check<a name="line.3370"></a>
<span class="sourceLineNo">3371</span>            assert Thread.holdsLock(this.raft);<a name="line.3371"></a>
<span class="sourceLineNo">3372</span><a name="line.3372"></a>
<span class="sourceLineNo">3373</span>            // Generate a randomized election timeout delay<a name="line.3373"></a>
<span class="sourceLineNo">3374</span>            final int range = this.raft.maxElectionTimeout - this.raft.minElectionTimeout;<a name="line.3374"></a>
<span class="sourceLineNo">3375</span>            final int randomizedPart = Math.round(this.raft.random.nextFloat() * range);<a name="line.3375"></a>
<span class="sourceLineNo">3376</span><a name="line.3376"></a>
<span class="sourceLineNo">3377</span>            // Restart timer<a name="line.3377"></a>
<span class="sourceLineNo">3378</span>            this.electionTimer.timeoutAfter(this.raft.minElectionTimeout + randomizedPart);<a name="line.3378"></a>
<span class="sourceLineNo">3379</span>        }<a name="line.3379"></a>
<span class="sourceLineNo">3380</span><a name="line.3380"></a>
<span class="sourceLineNo">3381</span>    // MessageSwitch<a name="line.3381"></a>
<span class="sourceLineNo">3382</span><a name="line.3382"></a>
<span class="sourceLineNo">3383</span>        @Override<a name="line.3383"></a>
<span class="sourceLineNo">3384</span>        public void caseAppendResponse(AppendResponse msg) {<a name="line.3384"></a>
<span class="sourceLineNo">3385</span>            this.failUnexpectedMessage(msg);<a name="line.3385"></a>
<span class="sourceLineNo">3386</span>        }<a name="line.3386"></a>
<span class="sourceLineNo">3387</span><a name="line.3387"></a>
<span class="sourceLineNo">3388</span>        @Override<a name="line.3388"></a>
<span class="sourceLineNo">3389</span>        public void caseCommitRequest(CommitRequest msg) {<a name="line.3389"></a>
<span class="sourceLineNo">3390</span>            this.failUnexpectedMessage(msg);<a name="line.3390"></a>
<span class="sourceLineNo">3391</span>        }<a name="line.3391"></a>
<span class="sourceLineNo">3392</span>    }<a name="line.3392"></a>
<span class="sourceLineNo">3393</span><a name="line.3393"></a>
<span class="sourceLineNo">3394</span>// FOLLOWER role<a name="line.3394"></a>
<span class="sourceLineNo">3395</span><a name="line.3395"></a>
<span class="sourceLineNo">3396</span>    private static class FollowerRole extends NonLeaderRole {<a name="line.3396"></a>
<span class="sourceLineNo">3397</span><a name="line.3397"></a>
<span class="sourceLineNo">3398</span>        private String leader;                                                          // our leader, if known<a name="line.3398"></a>
<span class="sourceLineNo">3399</span>        private String leaderAddress;                                                   // our leader's network address<a name="line.3399"></a>
<span class="sourceLineNo">3400</span>        private String votedFor;                                                        // the candidate we voted for this term<a name="line.3400"></a>
<span class="sourceLineNo">3401</span>        private SnapshotReceive snapshotReceive;                                        // in-progress snapshot install, if any<a name="line.3401"></a>
<span class="sourceLineNo">3402</span>        private final HashMap&lt;Long, PendingRequest&gt; pendingRequests = new HashMap&lt;&gt;();  // wait for CommitResponse or log entry<a name="line.3402"></a>
<span class="sourceLineNo">3403</span>        private final HashMap&lt;Long, PendingWrite&gt; pendingWrites = new HashMap&lt;&gt;();      // wait for AppendRequest with null data<a name="line.3403"></a>
<span class="sourceLineNo">3404</span>        private final HashMap&lt;Long, Timestamp&gt; commitLeaderLeaseTimeoutMap              // tx's waiting for leaderLeaseTimeout's<a name="line.3404"></a>
<span class="sourceLineNo">3405</span>          = new HashMap&lt;&gt;();<a name="line.3405"></a>
<span class="sourceLineNo">3406</span>        private Timestamp lastLeaderMessageTime;                                        // time of most recent rec'd AppendRequest<a name="line.3406"></a>
<span class="sourceLineNo">3407</span>        private Timestamp leaderLeaseTimeout;                                           // latest rec'd leader lease timeout<a name="line.3407"></a>
<span class="sourceLineNo">3408</span><a name="line.3408"></a>
<span class="sourceLineNo">3409</span>    // Constructors<a name="line.3409"></a>
<span class="sourceLineNo">3410</span><a name="line.3410"></a>
<span class="sourceLineNo">3411</span>        public FollowerRole(RaftKVDatabase raft) {<a name="line.3411"></a>
<span class="sourceLineNo">3412</span>            this(raft, null, null, null);<a name="line.3412"></a>
<span class="sourceLineNo">3413</span>        }<a name="line.3413"></a>
<span class="sourceLineNo">3414</span><a name="line.3414"></a>
<span class="sourceLineNo">3415</span>        public FollowerRole(RaftKVDatabase raft, String leader, String leaderAddress) {<a name="line.3415"></a>
<span class="sourceLineNo">3416</span>            this(raft, leader, leaderAddress, leader);<a name="line.3416"></a>
<span class="sourceLineNo">3417</span>        }<a name="line.3417"></a>
<span class="sourceLineNo">3418</span><a name="line.3418"></a>
<span class="sourceLineNo">3419</span>        public FollowerRole(RaftKVDatabase raft, String leader, String leaderAddress, String votedFor) {<a name="line.3419"></a>
<span class="sourceLineNo">3420</span>            super(raft, raft.isClusterMember());<a name="line.3420"></a>
<span class="sourceLineNo">3421</span>            this.leader = leader;<a name="line.3421"></a>
<span class="sourceLineNo">3422</span>            this.leaderAddress = leaderAddress;<a name="line.3422"></a>
<span class="sourceLineNo">3423</span>            this.votedFor = votedFor;<a name="line.3423"></a>
<span class="sourceLineNo">3424</span>            assert this.leaderAddress != null || this.leader == null;<a name="line.3424"></a>
<span class="sourceLineNo">3425</span>        }<a name="line.3425"></a>
<span class="sourceLineNo">3426</span><a name="line.3426"></a>
<span class="sourceLineNo">3427</span>    // Lifecycle<a name="line.3427"></a>
<span class="sourceLineNo">3428</span><a name="line.3428"></a>
<span class="sourceLineNo">3429</span>        @Override<a name="line.3429"></a>
<span class="sourceLineNo">3430</span>        public void setup() {<a name="line.3430"></a>
<span class="sourceLineNo">3431</span>            super.setup();<a name="line.3431"></a>
<span class="sourceLineNo">3432</span>            this.info("entering follower role in term " + this.raft.currentTerm<a name="line.3432"></a>
<span class="sourceLineNo">3433</span>              + (this.leader != null ? "; with leader \"" + this.leader + "\" at " + this.leaderAddress : "")<a name="line.3433"></a>
<span class="sourceLineNo">3434</span>              + (this.votedFor != null ? "; having voted for \"" + this.votedFor + "\"" : ""));<a name="line.3434"></a>
<span class="sourceLineNo">3435</span>        }<a name="line.3435"></a>
<span class="sourceLineNo">3436</span><a name="line.3436"></a>
<span class="sourceLineNo">3437</span>        @Override<a name="line.3437"></a>
<span class="sourceLineNo">3438</span>        public void shutdown() {<a name="line.3438"></a>
<span class="sourceLineNo">3439</span>            super.shutdown();<a name="line.3439"></a>
<span class="sourceLineNo">3440</span><a name="line.3440"></a>
<span class="sourceLineNo">3441</span>            // Cancel any in-progress snapshot install<a name="line.3441"></a>
<span class="sourceLineNo">3442</span>            if (this.snapshotReceive != null) {<a name="line.3442"></a>
<span class="sourceLineNo">3443</span>                if (this.log.isDebugEnabled())<a name="line.3443"></a>
<span class="sourceLineNo">3444</span>                    this.debug("aborting snapshot install due to leaving follower role");<a name="line.3444"></a>
<span class="sourceLineNo">3445</span>                this.raft.resetStateMachine(true);<a name="line.3445"></a>
<span class="sourceLineNo">3446</span>                this.snapshotReceive = null;<a name="line.3446"></a>
<span class="sourceLineNo">3447</span>            }<a name="line.3447"></a>
<span class="sourceLineNo">3448</span><a name="line.3448"></a>
<span class="sourceLineNo">3449</span>            // Fail any (read-only) transactions waiting on a minimum lease timeout from deposed leader<a name="line.3449"></a>
<span class="sourceLineNo">3450</span>            for (RaftKVTransaction tx : new ArrayList&lt;RaftKVTransaction&gt;(this.raft.openTransactions.values())) {<a name="line.3450"></a>
<span class="sourceLineNo">3451</span>                if (tx.getState().equals(TxState.COMMIT_WAITING) &amp;&amp; this.commitLeaderLeaseTimeoutMap.containsKey(tx.getTxId()))<a name="line.3451"></a>
<span class="sourceLineNo">3452</span>                    this.raft.fail(tx, new RetryTransactionException(tx, "leader was deposed during commit"));<a name="line.3452"></a>
<span class="sourceLineNo">3453</span>            }<a name="line.3453"></a>
<span class="sourceLineNo">3454</span><a name="line.3454"></a>
<span class="sourceLineNo">3455</span>            // Cleanup pending requests and commit writes<a name="line.3455"></a>
<span class="sourceLineNo">3456</span>            this.pendingRequests.clear();<a name="line.3456"></a>
<span class="sourceLineNo">3457</span>            for (PendingWrite pendingWrite : this.pendingWrites.values())<a name="line.3457"></a>
<span class="sourceLineNo">3458</span>                pendingWrite.cleanup();<a name="line.3458"></a>
<span class="sourceLineNo">3459</span>            this.pendingWrites.clear();<a name="line.3459"></a>
<span class="sourceLineNo">3460</span>        }<a name="line.3460"></a>
<span class="sourceLineNo">3461</span><a name="line.3461"></a>
<span class="sourceLineNo">3462</span>    // Service<a name="line.3462"></a>
<span class="sourceLineNo">3463</span><a name="line.3463"></a>
<span class="sourceLineNo">3464</span>        @Override<a name="line.3464"></a>
<span class="sourceLineNo">3465</span>        public void outputQueueEmpty(String address) {<a name="line.3465"></a>
<span class="sourceLineNo">3466</span>            if (address.equals(this.leaderAddress))<a name="line.3466"></a>
<span class="sourceLineNo">3467</span>                this.raft.requestService(this.checkReadyTransactionsService);       // TODO: track specific transactions<a name="line.3467"></a>
<span class="sourceLineNo">3468</span>        }<a name="line.3468"></a>
<span class="sourceLineNo">3469</span><a name="line.3469"></a>
<span class="sourceLineNo">3470</span>        // Check whether the required minimum leader lease timeout has been seen, if any<a name="line.3470"></a>
<span class="sourceLineNo">3471</span>        @Override<a name="line.3471"></a>
<span class="sourceLineNo">3472</span>        protected boolean mayCommit(RaftKVTransaction tx) {<a name="line.3472"></a>
<span class="sourceLineNo">3473</span><a name="line.3473"></a>
<span class="sourceLineNo">3474</span>            // Is there a required minimum leader lease timeout associated with the transaction?<a name="line.3474"></a>
<span class="sourceLineNo">3475</span>            final Timestamp commitLeaderLeaseTimeout = this.commitLeaderLeaseTimeoutMap.get(tx.getTxId());<a name="line.3475"></a>
<span class="sourceLineNo">3476</span>            if (commitLeaderLeaseTimeout == null)<a name="line.3476"></a>
<span class="sourceLineNo">3477</span>                return true;<a name="line.3477"></a>
<span class="sourceLineNo">3478</span><a name="line.3478"></a>
<span class="sourceLineNo">3479</span>            // Do we know the leader's lease timeout yet?<a name="line.3479"></a>
<span class="sourceLineNo">3480</span>            if (this.leaderLeaseTimeout == null)<a name="line.3480"></a>
<span class="sourceLineNo">3481</span>                return false;<a name="line.3481"></a>
<span class="sourceLineNo">3482</span><a name="line.3482"></a>
<span class="sourceLineNo">3483</span>            // Verify leader's lease timeout has extended beyond that required by the transaction<a name="line.3483"></a>
<span class="sourceLineNo">3484</span>            return this.leaderLeaseTimeout.compareTo(commitLeaderLeaseTimeout) &gt;= 0;<a name="line.3484"></a>
<span class="sourceLineNo">3485</span>        }<a name="line.3485"></a>
<span class="sourceLineNo">3486</span><a name="line.3486"></a>
<span class="sourceLineNo">3487</span>        /**<a name="line.3487"></a>
<span class="sourceLineNo">3488</span>         * Check whether the election timer should be running, and make it so.<a name="line.3488"></a>
<span class="sourceLineNo">3489</span>         *<a name="line.3489"></a>
<span class="sourceLineNo">3490</span>         * &lt;p&gt;<a name="line.3490"></a>
<span class="sourceLineNo">3491</span>         * This should be invoked:<a name="line.3491"></a>
<span class="sourceLineNo">3492</span>         * &lt;ul&gt;<a name="line.3492"></a>
<span class="sourceLineNo">3493</span>         *  &lt;li&gt;After a log entry that contains a configuration change has been added to the log&lt;/li&gt;<a name="line.3493"></a>
<span class="sourceLineNo">3494</span>         *  &lt;li&gt;When a snapshot install starts&lt;/li&gt;<a name="line.3494"></a>
<span class="sourceLineNo">3495</span>         *  &lt;li&gt;When a snapshot install completes&lt;/li&gt;<a name="line.3495"></a>
<span class="sourceLineNo">3496</span>         * &lt;/ul&gt;<a name="line.3496"></a>
<span class="sourceLineNo">3497</span>         */<a name="line.3497"></a>
<span class="sourceLineNo">3498</span>        private void updateElectionTimer() {<a name="line.3498"></a>
<span class="sourceLineNo">3499</span>            final boolean isClusterMember = this.raft.isClusterMember();<a name="line.3499"></a>
<span class="sourceLineNo">3500</span>            final boolean electionTimerRunning = this.electionTimer.isRunning();<a name="line.3500"></a>
<span class="sourceLineNo">3501</span>            if (isClusterMember &amp;&amp; !electionTimerRunning) {<a name="line.3501"></a>
<span class="sourceLineNo">3502</span>                if (this.log.isTraceEnabled())<a name="line.3502"></a>
<span class="sourceLineNo">3503</span>                    this.trace("starting up election timer because I'm now part of the current config");<a name="line.3503"></a>
<span class="sourceLineNo">3504</span>                this.restartElectionTimer();<a name="line.3504"></a>
<span class="sourceLineNo">3505</span>            } else if (!isClusterMember &amp;&amp; electionTimerRunning) {<a name="line.3505"></a>
<span class="sourceLineNo">3506</span>                if (this.log.isTraceEnabled())<a name="line.3506"></a>
<span class="sourceLineNo">3507</span>                    this.trace("stopping election timer because I'm no longer part of the current config");<a name="line.3507"></a>
<span class="sourceLineNo">3508</span>                this.electionTimer.cancel();<a name="line.3508"></a>
<span class="sourceLineNo">3509</span>            }<a name="line.3509"></a>
<span class="sourceLineNo">3510</span>        }<a name="line.3510"></a>
<span class="sourceLineNo">3511</span><a name="line.3511"></a>
<span class="sourceLineNo">3512</span>    // Transactions<a name="line.3512"></a>
<span class="sourceLineNo">3513</span><a name="line.3513"></a>
<span class="sourceLineNo">3514</span>        @Override<a name="line.3514"></a>
<span class="sourceLineNo">3515</span>        public void checkReadyTransaction(RaftKVTransaction tx) {<a name="line.3515"></a>
<span class="sourceLineNo">3516</span><a name="line.3516"></a>
<span class="sourceLineNo">3517</span>            // Sanity check<a name="line.3517"></a>
<span class="sourceLineNo">3518</span>            assert Thread.holdsLock(this.raft);<a name="line.3518"></a>
<span class="sourceLineNo">3519</span>            assert tx.getState().equals(TxState.COMMIT_READY);<a name="line.3519"></a>
<span class="sourceLineNo">3520</span><a name="line.3520"></a>
<span class="sourceLineNo">3521</span>            // Check snapshot isolation<a name="line.3521"></a>
<span class="sourceLineNo">3522</span>            if (this.checkReadyTransactionReadOnlySnapshot(tx))<a name="line.3522"></a>
<span class="sourceLineNo">3523</span>                return;<a name="line.3523"></a>
<span class="sourceLineNo">3524</span><a name="line.3524"></a>
<span class="sourceLineNo">3525</span>            // Did we already send a CommitRequest for this transaction?<a name="line.3525"></a>
<span class="sourceLineNo">3526</span>            PendingRequest pendingRequest = this.pendingRequests.get(tx.getTxId());<a name="line.3526"></a>
<span class="sourceLineNo">3527</span>            if (pendingRequest != null) {<a name="line.3527"></a>
<span class="sourceLineNo">3528</span>                if (this.log.isTraceEnabled())<a name="line.3528"></a>
<span class="sourceLineNo">3529</span>                    this.trace("leaving alone ready tx " + tx + " because request already sent");<a name="line.3529"></a>
<span class="sourceLineNo">3530</span>                return;<a name="line.3530"></a>
<span class="sourceLineNo">3531</span>            }<a name="line.3531"></a>
<span class="sourceLineNo">3532</span><a name="line.3532"></a>
<span class="sourceLineNo">3533</span>            // If we are installing a snapshot, we must wait<a name="line.3533"></a>
<span class="sourceLineNo">3534</span>            if (this.snapshotReceive != null) {<a name="line.3534"></a>
<span class="sourceLineNo">3535</span>                if (this.log.isTraceEnabled())<a name="line.3535"></a>
<span class="sourceLineNo">3536</span>                    this.trace("leaving alone ready tx " + tx + " because a snapshot install is in progress");<a name="line.3536"></a>
<span class="sourceLineNo">3537</span>                return;<a name="line.3537"></a>
<span class="sourceLineNo">3538</span>            }<a name="line.3538"></a>
<span class="sourceLineNo">3539</span><a name="line.3539"></a>
<span class="sourceLineNo">3540</span>            // Handle situation where we are unconfigured and not part of any cluster yet<a name="line.3540"></a>
<span class="sourceLineNo">3541</span>            if (!this.raft.isConfigured()) {<a name="line.3541"></a>
<span class="sourceLineNo">3542</span><a name="line.3542"></a>
<span class="sourceLineNo">3543</span>                // Get transaction mutations<a name="line.3543"></a>
<span class="sourceLineNo">3544</span>                final Writes writes = tx.getMutableView().getWrites();<a name="line.3544"></a>
<span class="sourceLineNo">3545</span>                final String[] configChange = tx.getConfigChange();<a name="line.3545"></a>
<span class="sourceLineNo">3546</span><a name="line.3546"></a>
<span class="sourceLineNo">3547</span>                // Allow an empty read-only transaction when unconfigured<a name="line.3547"></a>
<span class="sourceLineNo">3548</span>                if (writes.isEmpty() &amp;&amp; configChange == null) {<a name="line.3548"></a>
<span class="sourceLineNo">3549</span>                    this.raft.succeed(tx);<a name="line.3549"></a>
<span class="sourceLineNo">3550</span>                    return;<a name="line.3550"></a>
<span class="sourceLineNo">3551</span>                }<a name="line.3551"></a>
<span class="sourceLineNo">3552</span><a name="line.3552"></a>
<span class="sourceLineNo">3553</span>                // Otherwise, we can only handle an initial config change that is adding the local node<a name="line.3553"></a>
<span class="sourceLineNo">3554</span>                if (configChange == null || !configChange[0].equals(this.raft.identity) || configChange[1] == null) {<a name="line.3554"></a>
<span class="sourceLineNo">3555</span>                    throw new RetryTransactionException(tx, "unconfigured system: an initial configuration change adding"<a name="line.3555"></a>
<span class="sourceLineNo">3556</span>                      + " the local node (\"" + this.raft.identity + "\") as the first member of a new cluster is required");<a name="line.3556"></a>
<span class="sourceLineNo">3557</span>                }<a name="line.3557"></a>
<span class="sourceLineNo">3558</span><a name="line.3558"></a>
<span class="sourceLineNo">3559</span>                // Create a new cluster if needed<a name="line.3559"></a>
<span class="sourceLineNo">3560</span>                if (this.raft.clusterId == 0) {<a name="line.3560"></a>
<span class="sourceLineNo">3561</span>                    this.info("creating new cluster");<a name="line.3561"></a>
<span class="sourceLineNo">3562</span>                    if (!this.raft.joinCluster(0))<a name="line.3562"></a>
<span class="sourceLineNo">3563</span>                        throw new KVTransactionException(tx, "error persisting new cluster ID");<a name="line.3563"></a>
<span class="sourceLineNo">3564</span>                }<a name="line.3564"></a>
<span class="sourceLineNo">3565</span><a name="line.3565"></a>
<span class="sourceLineNo">3566</span>                // Advance term<a name="line.3566"></a>
<span class="sourceLineNo">3567</span>                assert this.raft.currentTerm == 0;<a name="line.3567"></a>
<span class="sourceLineNo">3568</span>                if (!this.raft.advanceTerm(this.raft.currentTerm + 1))<a name="line.3568"></a>
<span class="sourceLineNo">3569</span>                    throw new KVTransactionException(tx, "error advancing term");<a name="line.3569"></a>
<span class="sourceLineNo">3570</span><a name="line.3570"></a>
<span class="sourceLineNo">3571</span>                // Append the first entry to the Raft log<a name="line.3571"></a>
<span class="sourceLineNo">3572</span>                final LogEntry logEntry;<a name="line.3572"></a>
<span class="sourceLineNo">3573</span>                try {<a name="line.3573"></a>
<span class="sourceLineNo">3574</span>                    logEntry = this.raft.appendLogEntry(this.raft.currentTerm, this.raft.new NewLogEntry(tx));<a name="line.3574"></a>
<span class="sourceLineNo">3575</span>                } catch (Exception e) {<a name="line.3575"></a>
<span class="sourceLineNo">3576</span>                    throw new KVTransactionException(tx, "error attempting to persist transaction", e);<a name="line.3576"></a>
<span class="sourceLineNo">3577</span>                }<a name="line.3577"></a>
<span class="sourceLineNo">3578</span>                if (this.log.isDebugEnabled())<a name="line.3578"></a>
<span class="sourceLineNo">3579</span>                    this.debug("added log entry " + logEntry + " for local transaction " + tx);<a name="line.3579"></a>
<span class="sourceLineNo">3580</span>                assert logEntry.getTerm() == 1;<a name="line.3580"></a>
<span class="sourceLineNo">3581</span>                assert logEntry.getIndex() == 1;<a name="line.3581"></a>
<span class="sourceLineNo">3582</span><a name="line.3582"></a>
<span class="sourceLineNo">3583</span>                // Set commit term and index from new log entry<a name="line.3583"></a>
<span class="sourceLineNo">3584</span>                tx.setCommitTerm(logEntry.getTerm());<a name="line.3584"></a>
<span class="sourceLineNo">3585</span>                tx.setCommitIndex(logEntry.getIndex());<a name="line.3585"></a>
<span class="sourceLineNo">3586</span>                this.raft.commitIndex = logEntry.getIndex();<a name="line.3586"></a>
<span class="sourceLineNo">3587</span><a name="line.3587"></a>
<span class="sourceLineNo">3588</span>                // Update transaction state<a name="line.3588"></a>
<span class="sourceLineNo">3589</span>                tx.setState(TxState.COMMIT_WAITING);<a name="line.3589"></a>
<span class="sourceLineNo">3590</span>                tx.getMutableView().disableReadTracking();                                  // we no longer need this info<a name="line.3590"></a>
<span class="sourceLineNo">3591</span><a name="line.3591"></a>
<span class="sourceLineNo">3592</span>                // Immediately become the leader of our new single-node cluster<a name="line.3592"></a>
<span class="sourceLineNo">3593</span>                assert this.raft.isConfigured();<a name="line.3593"></a>
<span class="sourceLineNo">3594</span>                this.info("appointing myself leader in newly created cluster");<a name="line.3594"></a>
<span class="sourceLineNo">3595</span>                this.raft.changeRole(new LeaderRole(this.raft));<a name="line.3595"></a>
<span class="sourceLineNo">3596</span>                return;<a name="line.3596"></a>
<span class="sourceLineNo">3597</span>            }<a name="line.3597"></a>
<span class="sourceLineNo">3598</span><a name="line.3598"></a>
<span class="sourceLineNo">3599</span>            // If we don't have a leader yet, or leader's queue is full, we must wait<a name="line.3599"></a>
<span class="sourceLineNo">3600</span>            if (this.leader == null || this.raft.isTransmitting(this.leaderAddress)) {<a name="line.3600"></a>
<span class="sourceLineNo">3601</span>                if (this.log.isTraceEnabled()) {<a name="line.3601"></a>
<span class="sourceLineNo">3602</span>                    this.trace("leaving alone ready tx " + tx + " because leader "<a name="line.3602"></a>
<span class="sourceLineNo">3603</span>                      + (this.leader == null ? "is not known yet" : "\"" + this.leader + "\" is not writable yet"));<a name="line.3603"></a>
<span class="sourceLineNo">3604</span>                }<a name="line.3604"></a>
<span class="sourceLineNo">3605</span>                return;<a name="line.3605"></a>
<span class="sourceLineNo">3606</span>            }<a name="line.3606"></a>
<span class="sourceLineNo">3607</span><a name="line.3607"></a>
<span class="sourceLineNo">3608</span>            // Serialize reads into buffer<a name="line.3608"></a>
<span class="sourceLineNo">3609</span>            final Reads reads = tx.getMutableView().getReads();<a name="line.3609"></a>
<span class="sourceLineNo">3610</span>            final long readsDataSize = reads.serializedLength();<a name="line.3610"></a>
<span class="sourceLineNo">3611</span>            if (readsDataSize != (int)readsDataSize)<a name="line.3611"></a>
<span class="sourceLineNo">3612</span>                throw new KVTransactionException(tx, "transaction read information exceeds maximum length");<a name="line.3612"></a>
<span class="sourceLineNo">3613</span>            final ByteBuffer readsData = Util.allocateByteBuffer((int)readsDataSize);<a name="line.3613"></a>
<span class="sourceLineNo">3614</span>            try (ByteBufferOutputStream output = new ByteBufferOutputStream(readsData)) {<a name="line.3614"></a>
<span class="sourceLineNo">3615</span>                reads.serialize(output);<a name="line.3615"></a>
<span class="sourceLineNo">3616</span>            } catch (IOException e) {<a name="line.3616"></a>
<span class="sourceLineNo">3617</span>                throw new RuntimeException("unexpected exception", e);<a name="line.3617"></a>
<span class="sourceLineNo">3618</span>            }<a name="line.3618"></a>
<span class="sourceLineNo">3619</span>            assert !readsData.hasRemaining();<a name="line.3619"></a>
<span class="sourceLineNo">3620</span>            readsData.flip();<a name="line.3620"></a>
<span class="sourceLineNo">3621</span><a name="line.3621"></a>
<span class="sourceLineNo">3622</span>            // Handle read-only vs. read-write transaction<a name="line.3622"></a>
<span class="sourceLineNo">3623</span>            final Writes writes = tx.getMutableView().getWrites();<a name="line.3623"></a>
<span class="sourceLineNo">3624</span>            FileWriter fileWriter = null;<a name="line.3624"></a>
<span class="sourceLineNo">3625</span>            ByteBuffer mutationData = null;<a name="line.3625"></a>
<span class="sourceLineNo">3626</span>            if (!writes.isEmpty() || tx.getConfigChange() != null) {<a name="line.3626"></a>
<span class="sourceLineNo">3627</span><a name="line.3627"></a>
<span class="sourceLineNo">3628</span>                // Serialize changes into a temporary file (but do not close or durably persist yet)<a name="line.3628"></a>
<span class="sourceLineNo">3629</span>                final File file = new File(this.raft.logDir,<a name="line.3629"></a>
<span class="sourceLineNo">3630</span>                  String.format("%s%019d%s", TX_FILE_PREFIX, tx.getTxId(), TEMP_FILE_SUFFIX));<a name="line.3630"></a>
<span class="sourceLineNo">3631</span>                try {<a name="line.3631"></a>
<span class="sourceLineNo">3632</span>                    LogEntry.writeData((fileWriter = new FileWriter(file)), new LogEntry.Data(writes, tx.getConfigChange()));<a name="line.3632"></a>
<span class="sourceLineNo">3633</span>                    fileWriter.flush();<a name="line.3633"></a>
<span class="sourceLineNo">3634</span>                } catch (IOException e) {<a name="line.3634"></a>
<span class="sourceLineNo">3635</span>                    fileWriter.getFile().delete();<a name="line.3635"></a>
<span class="sourceLineNo">3636</span>                    Util.closeIfPossible(fileWriter);<a name="line.3636"></a>
<span class="sourceLineNo">3637</span>                    throw new KVTransactionException(tx, "error saving transaction mutations to temporary file", e);<a name="line.3637"></a>
<span class="sourceLineNo">3638</span>                }<a name="line.3638"></a>
<span class="sourceLineNo">3639</span>                final long writeLength = fileWriter.getLength();<a name="line.3639"></a>
<span class="sourceLineNo">3640</span><a name="line.3640"></a>
<span class="sourceLineNo">3641</span>                // Load serialized writes from file<a name="line.3641"></a>
<span class="sourceLineNo">3642</span>                try {<a name="line.3642"></a>
<span class="sourceLineNo">3643</span>                    mutationData = Util.readFile(fileWriter.getFile(), writeLength);<a name="line.3643"></a>
<span class="sourceLineNo">3644</span>                } catch (IOException e) {<a name="line.3644"></a>
<span class="sourceLineNo">3645</span>                    fileWriter.getFile().delete();<a name="line.3645"></a>
<span class="sourceLineNo">3646</span>                    Util.closeIfPossible(fileWriter);<a name="line.3646"></a>
<span class="sourceLineNo">3647</span>                    throw new KVTransactionException(tx, "error reading transaction mutations from temporary file", e);<a name="line.3647"></a>
<span class="sourceLineNo">3648</span>                }<a name="line.3648"></a>
<span class="sourceLineNo">3649</span><a name="line.3649"></a>
<span class="sourceLineNo">3650</span>                // Record pending commit write with temporary file<a name="line.3650"></a>
<span class="sourceLineNo">3651</span>                final PendingWrite pendingWrite = new PendingWrite(tx, fileWriter);<a name="line.3651"></a>
<span class="sourceLineNo">3652</span>                this.pendingWrites.put(tx.getTxId(), pendingWrite);<a name="line.3652"></a>
<span class="sourceLineNo">3653</span>            }<a name="line.3653"></a>
<span class="sourceLineNo">3654</span><a name="line.3654"></a>
<span class="sourceLineNo">3655</span>            // Record pending request<a name="line.3655"></a>
<span class="sourceLineNo">3656</span>            this.pendingRequests.put(tx.getTxId(), new PendingRequest(tx));<a name="line.3656"></a>
<span class="sourceLineNo">3657</span><a name="line.3657"></a>
<span class="sourceLineNo">3658</span>            // Send commit request to leader<a name="line.3658"></a>
<span class="sourceLineNo">3659</span>            final CommitRequest msg = new CommitRequest(this.raft.clusterId, this.raft.identity, this.leader,<a name="line.3659"></a>
<span class="sourceLineNo">3660</span>              this.raft.currentTerm, tx.getTxId(), tx.getBaseTerm(), tx.getBaseIndex(), readsData, mutationData);<a name="line.3660"></a>
<span class="sourceLineNo">3661</span>            if (this.log.isTraceEnabled())<a name="line.3661"></a>
<span class="sourceLineNo">3662</span>                this.trace("sending " + msg + " to \"" + this.leader + "\" for " + tx);<a name="line.3662"></a>
<span class="sourceLineNo">3663</span>            if (!this.raft.sendMessage(msg))<a name="line.3663"></a>
<span class="sourceLineNo">3664</span>                throw new RetryTransactionException(tx, "error sending commit request to leader");<a name="line.3664"></a>
<span class="sourceLineNo">3665</span>        }<a name="line.3665"></a>
<span class="sourceLineNo">3666</span><a name="line.3666"></a>
<span class="sourceLineNo">3667</span>        @Override<a name="line.3667"></a>
<span class="sourceLineNo">3668</span>        public void cleanupForTransaction(RaftKVTransaction tx) {<a name="line.3668"></a>
<span class="sourceLineNo">3669</span>            this.pendingRequests.remove(tx.getTxId());<a name="line.3669"></a>
<span class="sourceLineNo">3670</span>            final PendingWrite pendingWrite = this.pendingWrites.remove(tx.getTxId());<a name="line.3670"></a>
<span class="sourceLineNo">3671</span>            if (pendingWrite != null)<a name="line.3671"></a>
<span class="sourceLineNo">3672</span>                pendingWrite.cleanup();<a name="line.3672"></a>
<span class="sourceLineNo">3673</span>            this.commitLeaderLeaseTimeoutMap.remove(tx.getTxId());<a name="line.3673"></a>
<span class="sourceLineNo">3674</span>        }<a name="line.3674"></a>
<span class="sourceLineNo">3675</span><a name="line.3675"></a>
<span class="sourceLineNo">3676</span>    // Messages<a name="line.3676"></a>
<span class="sourceLineNo">3677</span><a name="line.3677"></a>
<span class="sourceLineNo">3678</span>        @Override<a name="line.3678"></a>
<span class="sourceLineNo">3679</span>        public boolean mayAdvanceCurrentTerm(Message msg) {<a name="line.3679"></a>
<span class="sourceLineNo">3680</span><a name="line.3680"></a>
<span class="sourceLineNo">3681</span>            // Deny vote if we have heard from our leader within the minimum election timeout (dissertation, section 4.2.3)<a name="line.3681"></a>
<span class="sourceLineNo">3682</span>            if (msg instanceof RequestVote<a name="line.3682"></a>
<span class="sourceLineNo">3683</span>              &amp;&amp; this.lastLeaderMessageTime != null<a name="line.3683"></a>
<span class="sourceLineNo">3684</span>              &amp;&amp; this.lastLeaderMessageTime.offsetFromNow() &gt; -this.raft.minElectionTimeout)<a name="line.3684"></a>
<span class="sourceLineNo">3685</span>                return false;<a name="line.3685"></a>
<span class="sourceLineNo">3686</span><a name="line.3686"></a>
<span class="sourceLineNo">3687</span>            // OK<a name="line.3687"></a>
<span class="sourceLineNo">3688</span>            return true;<a name="line.3688"></a>
<span class="sourceLineNo">3689</span>        }<a name="line.3689"></a>
<span class="sourceLineNo">3690</span><a name="line.3690"></a>
<span class="sourceLineNo">3691</span>        @Override<a name="line.3691"></a>
<span class="sourceLineNo">3692</span>        public void caseAppendRequest(AppendRequest msg) {<a name="line.3692"></a>
<span class="sourceLineNo">3693</span><a name="line.3693"></a>
<span class="sourceLineNo">3694</span>            // Record new cluster ID if we haven't done so already<a name="line.3694"></a>
<span class="sourceLineNo">3695</span>            if (this.raft.clusterId == 0)<a name="line.3695"></a>
<span class="sourceLineNo">3696</span>                this.raft.joinCluster(msg.getClusterId());<a name="line.3696"></a>
<span class="sourceLineNo">3697</span><a name="line.3697"></a>
<span class="sourceLineNo">3698</span>            // Record leader<a name="line.3698"></a>
<span class="sourceLineNo">3699</span>            if (!msg.getSenderId().equals(this.leader)) {<a name="line.3699"></a>
<span class="sourceLineNo">3700</span>                if (this.leader != null &amp;&amp; !this.leader.equals(msg.getSenderId())) {<a name="line.3700"></a>
<span class="sourceLineNo">3701</span>                    this.error("detected a conflicting leader in " + msg + " (previous leader was \"" + this.leader<a name="line.3701"></a>
<span class="sourceLineNo">3702</span>                      + "\") - should never happen; possible inconsistent cluster configuration (mine: " + this.raft.currentConfig<a name="line.3702"></a>
<span class="sourceLineNo">3703</span>                      + ")");<a name="line.3703"></a>
<span class="sourceLineNo">3704</span>                }<a name="line.3704"></a>
<span class="sourceLineNo">3705</span>                this.leader = msg.getSenderId();<a name="line.3705"></a>
<span class="sourceLineNo">3706</span>                this.leaderAddress = this.raft.returnAddress;<a name="line.3706"></a>
<span class="sourceLineNo">3707</span>                this.leaderLeaseTimeout = msg.getLeaderLeaseTimeout();<a name="line.3707"></a>
<span class="sourceLineNo">3708</span>                this.info("updated leader to \"" + this.leader + "\" at " + this.leaderAddress);<a name="line.3708"></a>
<span class="sourceLineNo">3709</span>                this.raft.requestService(this.checkReadyTransactionsService);     // allows COMMIT_READY transactions to be sent<a name="line.3709"></a>
<span class="sourceLineNo">3710</span>            }<a name="line.3710"></a>
<span class="sourceLineNo">3711</span><a name="line.3711"></a>
<span class="sourceLineNo">3712</span>            // Get message info<a name="line.3712"></a>
<span class="sourceLineNo">3713</span>            final long leaderCommitIndex = msg.getLeaderCommit();<a name="line.3713"></a>
<span class="sourceLineNo">3714</span>            final long leaderPrevTerm = msg.getPrevLogTerm();<a name="line.3714"></a>
<span class="sourceLineNo">3715</span>            final long leaderPrevIndex = msg.getPrevLogIndex();<a name="line.3715"></a>
<span class="sourceLineNo">3716</span>            final long logTerm = msg.getLogEntryTerm();<a name="line.3716"></a>
<span class="sourceLineNo">3717</span>            final long logIndex = leaderPrevIndex + 1;<a name="line.3717"></a>
<span class="sourceLineNo">3718</span><a name="line.3718"></a>
<span class="sourceLineNo">3719</span>            // Update timestamp last heard from leader<a name="line.3719"></a>
<span class="sourceLineNo">3720</span>            this.lastLeaderMessageTime = new Timestamp();<a name="line.3720"></a>
<span class="sourceLineNo">3721</span><a name="line.3721"></a>
<span class="sourceLineNo">3722</span>            // Update leader's lease timeout<a name="line.3722"></a>
<span class="sourceLineNo">3723</span>            if (this.leaderLeaseTimeout == null || msg.getLeaderLeaseTimeout().compareTo(this.leaderLeaseTimeout) &gt; 0) {<a name="line.3723"></a>
<span class="sourceLineNo">3724</span>                if (this.log.isTraceEnabled())<a name="line.3724"></a>
<span class="sourceLineNo">3725</span>                    this.trace("advancing leader lease timeout " + this.leaderLeaseTimeout + " -&gt; " + msg.getLeaderLeaseTimeout());<a name="line.3725"></a>
<span class="sourceLineNo">3726</span>                this.leaderLeaseTimeout = msg.getLeaderLeaseTimeout();<a name="line.3726"></a>
<span class="sourceLineNo">3727</span>                this.raft.requestService(this.checkWaitingTransactionsService);<a name="line.3727"></a>
<span class="sourceLineNo">3728</span>            }<a name="line.3728"></a>
<span class="sourceLineNo">3729</span><a name="line.3729"></a>
<span class="sourceLineNo">3730</span>            // If a snapshot install is in progress, cancel it<a name="line.3730"></a>
<span class="sourceLineNo">3731</span>            if (this.snapshotReceive != null) {<a name="line.3731"></a>
<span class="sourceLineNo">3732</span>                this.info("rec'd " + msg + " during in-progress " + this.snapshotReceive<a name="line.3732"></a>
<span class="sourceLineNo">3733</span>                  + "; aborting snapshot install and resetting state machine");<a name="line.3733"></a>
<span class="sourceLineNo">3734</span>                if (!this.raft.resetStateMachine(true))<a name="line.3734"></a>
<span class="sourceLineNo">3735</span>                    return;<a name="line.3735"></a>
<span class="sourceLineNo">3736</span>                this.snapshotReceive = null;<a name="line.3736"></a>
<span class="sourceLineNo">3737</span>                this.updateElectionTimer();<a name="line.3737"></a>
<span class="sourceLineNo">3738</span>            }<a name="line.3738"></a>
<span class="sourceLineNo">3739</span><a name="line.3739"></a>
<span class="sourceLineNo">3740</span>            // Restart election timeout (if running)<a name="line.3740"></a>
<span class="sourceLineNo">3741</span>            if (this.electionTimer.isRunning())<a name="line.3741"></a>
<span class="sourceLineNo">3742</span>                this.restartElectionTimer();<a name="line.3742"></a>
<span class="sourceLineNo">3743</span><a name="line.3743"></a>
<span class="sourceLineNo">3744</span>            // Get my last log entry's index and term<a name="line.3744"></a>
<span class="sourceLineNo">3745</span>            long lastLogTerm = this.raft.getLastLogTerm();<a name="line.3745"></a>
<span class="sourceLineNo">3746</span>            long lastLogIndex = this.raft.getLastLogIndex();<a name="line.3746"></a>
<span class="sourceLineNo">3747</span><a name="line.3747"></a>
<span class="sourceLineNo">3748</span>            // Check whether our previous log entry term matches that of leader; if not, or it doesn't exist, request fails<a name="line.3748"></a>
<span class="sourceLineNo">3749</span>            if (leaderPrevIndex &lt; this.raft.lastAppliedIndex<a name="line.3749"></a>
<span class="sourceLineNo">3750</span>              || leaderPrevIndex &gt; lastLogIndex<a name="line.3750"></a>
<span class="sourceLineNo">3751</span>              || leaderPrevTerm != this.raft.getLogTermAtIndex(leaderPrevIndex)) {<a name="line.3751"></a>
<span class="sourceLineNo">3752</span>                if (this.log.isDebugEnabled())<a name="line.3752"></a>
<span class="sourceLineNo">3753</span>                    this.debug("rejecting " + msg + " because previous log entry doesn't match");<a name="line.3753"></a>
<span class="sourceLineNo">3754</span>                this.raft.sendMessage(new AppendResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3754"></a>
<span class="sourceLineNo">3755</span>                  this.raft.currentTerm, msg.getLeaderTimestamp(), false, this.raft.lastAppliedIndex, this.raft.getLastLogIndex()));<a name="line.3755"></a>
<span class="sourceLineNo">3756</span>                return;<a name="line.3756"></a>
<span class="sourceLineNo">3757</span>            }<a name="line.3757"></a>
<span class="sourceLineNo">3758</span><a name="line.3758"></a>
<span class="sourceLineNo">3759</span>            // Check whether the message actually contains a log entry; if so, append it<a name="line.3759"></a>
<span class="sourceLineNo">3760</span>            boolean success = true;<a name="line.3760"></a>
<span class="sourceLineNo">3761</span>            if (!msg.isProbe()) {<a name="line.3761"></a>
<span class="sourceLineNo">3762</span><a name="line.3762"></a>
<span class="sourceLineNo">3763</span>                // Check for a conflicting (i.e., never committed, then overwritten) log entry that we need to clear away first<a name="line.3763"></a>
<span class="sourceLineNo">3764</span>                if (logIndex &lt;= lastLogIndex &amp;&amp; this.raft.getLogTermAtIndex(logIndex) != msg.getLogEntryTerm()) {<a name="line.3764"></a>
<span class="sourceLineNo">3765</span><a name="line.3765"></a>
<span class="sourceLineNo">3766</span>                    // Delete conflicting log entry, and all entries that follow it, from the log<a name="line.3766"></a>
<span class="sourceLineNo">3767</span>                    final int startListIndex = (int)(logIndex - this.raft.lastAppliedIndex - 1);<a name="line.3767"></a>
<span class="sourceLineNo">3768</span>                    final List&lt;LogEntry&gt; conflictList = this.raft.raftLog.subList(startListIndex, this.raft.raftLog.size());<a name="line.3768"></a>
<span class="sourceLineNo">3769</span>                    for (LogEntry logEntry : conflictList) {<a name="line.3769"></a>
<span class="sourceLineNo">3770</span>                        this.info("deleting log entry " + logEntry + " overrwritten by " + msg);<a name="line.3770"></a>
<span class="sourceLineNo">3771</span>                        if (!logEntry.getFile().delete())<a name="line.3771"></a>
<span class="sourceLineNo">3772</span>                            this.error("failed to delete log file " + logEntry.getFile());<a name="line.3772"></a>
<span class="sourceLineNo">3773</span>                    }<a name="line.3773"></a>
<span class="sourceLineNo">3774</span>                    try {<a name="line.3774"></a>
<span class="sourceLineNo">3775</span>                        this.raft.logDirChannel.force(true);<a name="line.3775"></a>
<span class="sourceLineNo">3776</span>                    } catch (IOException e) {<a name="line.3776"></a>
<span class="sourceLineNo">3777</span>                        this.warn("errory fsync()'ing log directory " + this.raft.logDir, e);<a name="line.3777"></a>
<span class="sourceLineNo">3778</span>                    }<a name="line.3778"></a>
<span class="sourceLineNo">3779</span>                    conflictList.clear();<a name="line.3779"></a>
<span class="sourceLineNo">3780</span><a name="line.3780"></a>
<span class="sourceLineNo">3781</span>                    // Rebuild current config<a name="line.3781"></a>
<span class="sourceLineNo">3782</span>                    this.raft.currentConfig = this.raft.buildCurrentConfig();<a name="line.3782"></a>
<span class="sourceLineNo">3783</span><a name="line.3783"></a>
<span class="sourceLineNo">3784</span>                    // Update last log entry info<a name="line.3784"></a>
<span class="sourceLineNo">3785</span>                    lastLogTerm = this.raft.getLastLogTerm();<a name="line.3785"></a>
<span class="sourceLineNo">3786</span>                    lastLogIndex = this.raft.getLastLogIndex();<a name="line.3786"></a>
<span class="sourceLineNo">3787</span>                }<a name="line.3787"></a>
<span class="sourceLineNo">3788</span><a name="line.3788"></a>
<span class="sourceLineNo">3789</span>                // Append the new log entry - if we don't already have it<a name="line.3789"></a>
<span class="sourceLineNo">3790</span>                if (logIndex &gt; lastLogIndex) {<a name="line.3790"></a>
<span class="sourceLineNo">3791</span>                    assert logIndex == lastLogIndex + 1;<a name="line.3791"></a>
<span class="sourceLineNo">3792</span>                    LogEntry logEntry = null;<a name="line.3792"></a>
<span class="sourceLineNo">3793</span>                    do {<a name="line.3793"></a>
<span class="sourceLineNo">3794</span><a name="line.3794"></a>
<span class="sourceLineNo">3795</span>                        // If message contains no data, we expect to get the data from the corresponding transaction<a name="line.3795"></a>
<span class="sourceLineNo">3796</span>                        final ByteBuffer mutationData = msg.getMutationData();<a name="line.3796"></a>
<span class="sourceLineNo">3797</span>                        if (mutationData == null) {<a name="line.3797"></a>
<span class="sourceLineNo">3798</span><a name="line.3798"></a>
<span class="sourceLineNo">3799</span>                            // Find the matching pending commit write<a name="line.3799"></a>
<span class="sourceLineNo">3800</span>                            final PendingWrite pendingWrite;<a name="line.3800"></a>
<span class="sourceLineNo">3801</span>                            try {<a name="line.3801"></a>
<span class="sourceLineNo">3802</span>                                pendingWrite = Iterables.find(this.pendingWrites.values(), new Predicate&lt;PendingWrite&gt;() {<a name="line.3802"></a>
<span class="sourceLineNo">3803</span>                                    @Override<a name="line.3803"></a>
<span class="sourceLineNo">3804</span>                                    public boolean apply(PendingWrite pendingWrite) {<a name="line.3804"></a>
<span class="sourceLineNo">3805</span>                                        final RaftKVTransaction tx = pendingWrite.getTx();<a name="line.3805"></a>
<span class="sourceLineNo">3806</span>                                        return tx.getState().equals(TxState.COMMIT_WAITING)<a name="line.3806"></a>
<span class="sourceLineNo">3807</span>                                          &amp;&amp; tx.getCommitTerm() == logTerm &amp;&amp; tx.getCommitIndex() == logIndex;<a name="line.3807"></a>
<span class="sourceLineNo">3808</span>                                    }<a name="line.3808"></a>
<span class="sourceLineNo">3809</span>                                });<a name="line.3809"></a>
<span class="sourceLineNo">3810</span>                            } catch (NoSuchElementException e) {<a name="line.3810"></a>
<span class="sourceLineNo">3811</span>                                if (this.log.isDebugEnabled()) {<a name="line.3811"></a>
<span class="sourceLineNo">3812</span>                                    this.debug("rec'd " + msg + " but no read-write transaction matching commit "<a name="line.3812"></a>
<span class="sourceLineNo">3813</span>                                      + logTerm + "/" + logIndex + " found; rejecting");<a name="line.3813"></a>
<span class="sourceLineNo">3814</span>                                }<a name="line.3814"></a>
<span class="sourceLineNo">3815</span>                                break;<a name="line.3815"></a>
<span class="sourceLineNo">3816</span>                            }<a name="line.3816"></a>
<span class="sourceLineNo">3817</span><a name="line.3817"></a>
<span class="sourceLineNo">3818</span>                            // Commit's writes are no longer pending<a name="line.3818"></a>
<span class="sourceLineNo">3819</span>                            final RaftKVTransaction tx = pendingWrite.getTx();<a name="line.3819"></a>
<span class="sourceLineNo">3820</span>                            this.pendingWrites.remove(tx.getTxId());<a name="line.3820"></a>
<span class="sourceLineNo">3821</span><a name="line.3821"></a>
<span class="sourceLineNo">3822</span>                            // Close and durably persist the associated temporary file<a name="line.3822"></a>
<span class="sourceLineNo">3823</span>                            try {<a name="line.3823"></a>
<span class="sourceLineNo">3824</span>                                pendingWrite.getFileWriter().close();<a name="line.3824"></a>
<span class="sourceLineNo">3825</span>                            } catch (IOException e) {<a name="line.3825"></a>
<span class="sourceLineNo">3826</span>                                this.error("error closing temporary transaction file for " + tx, e);<a name="line.3826"></a>
<span class="sourceLineNo">3827</span>                                pendingWrite.cleanup();<a name="line.3827"></a>
<span class="sourceLineNo">3828</span>                                break;<a name="line.3828"></a>
<span class="sourceLineNo">3829</span>                            }<a name="line.3829"></a>
<span class="sourceLineNo">3830</span><a name="line.3830"></a>
<span class="sourceLineNo">3831</span>                            // Append a new log entry using temporary file<a name="line.3831"></a>
<span class="sourceLineNo">3832</span>                            try {<a name="line.3832"></a>
<span class="sourceLineNo">3833</span>                                logEntry = this.raft.appendLogEntry(logTerm,<a name="line.3833"></a>
<span class="sourceLineNo">3834</span>                                  this.raft.new NewLogEntry(tx, pendingWrite.getFileWriter().getFile()));<a name="line.3834"></a>
<span class="sourceLineNo">3835</span>                            } catch (Exception e) {<a name="line.3835"></a>
<span class="sourceLineNo">3836</span>                                this.error("error appending new log entry for " + tx, e);<a name="line.3836"></a>
<span class="sourceLineNo">3837</span>                                pendingWrite.cleanup();<a name="line.3837"></a>
<span class="sourceLineNo">3838</span>                                break;<a name="line.3838"></a>
<span class="sourceLineNo">3839</span>                            }<a name="line.3839"></a>
<span class="sourceLineNo">3840</span><a name="line.3840"></a>
<span class="sourceLineNo">3841</span>                            // Debug<a name="line.3841"></a>
<span class="sourceLineNo">3842</span>                            if (this.log.isDebugEnabled()) {<a name="line.3842"></a>
<span class="sourceLineNo">3843</span>                                this.debug("now waiting for commit of " + tx.getCommitTerm() + "/" + tx.getCommitIndex()<a name="line.3843"></a>
<span class="sourceLineNo">3844</span>                                  + " to commit " + tx);<a name="line.3844"></a>
<span class="sourceLineNo">3845</span>                            }<a name="line.3845"></a>
<span class="sourceLineNo">3846</span>                        } else {<a name="line.3846"></a>
<span class="sourceLineNo">3847</span><a name="line.3847"></a>
<span class="sourceLineNo">3848</span>                            // Append new log entry normally using the data from the request<a name="line.3848"></a>
<span class="sourceLineNo">3849</span>                            try {<a name="line.3849"></a>
<span class="sourceLineNo">3850</span>                                logEntry = this.raft.appendLogEntry(logTerm, this.raft.new NewLogEntry(mutationData));<a name="line.3850"></a>
<span class="sourceLineNo">3851</span>                            } catch (Exception e) {<a name="line.3851"></a>
<span class="sourceLineNo">3852</span>                                this.error("error appending new log entry", e);<a name="line.3852"></a>
<span class="sourceLineNo">3853</span>                                break;<a name="line.3853"></a>
<span class="sourceLineNo">3854</span>                            }<a name="line.3854"></a>
<span class="sourceLineNo">3855</span>                        }<a name="line.3855"></a>
<span class="sourceLineNo">3856</span>                    } while (false);<a name="line.3856"></a>
<span class="sourceLineNo">3857</span><a name="line.3857"></a>
<span class="sourceLineNo">3858</span>                    // Start/stop election timer as needed<a name="line.3858"></a>
<span class="sourceLineNo">3859</span>                    if (logEntry != null &amp;&amp; logEntry.getConfigChange() != null)<a name="line.3859"></a>
<span class="sourceLineNo">3860</span>                        this.updateElectionTimer();<a name="line.3860"></a>
<span class="sourceLineNo">3861</span><a name="line.3861"></a>
<span class="sourceLineNo">3862</span>                    // Success?<a name="line.3862"></a>
<span class="sourceLineNo">3863</span>                    success = logEntry != null;<a name="line.3863"></a>
<span class="sourceLineNo">3864</span><a name="line.3864"></a>
<span class="sourceLineNo">3865</span>                    // Update last log entry info<a name="line.3865"></a>
<span class="sourceLineNo">3866</span>                    lastLogTerm = this.raft.getLastLogTerm();<a name="line.3866"></a>
<span class="sourceLineNo">3867</span>                    lastLogIndex = this.raft.getLastLogIndex();<a name="line.3867"></a>
<span class="sourceLineNo">3868</span>                }<a name="line.3868"></a>
<span class="sourceLineNo">3869</span>            }<a name="line.3869"></a>
<span class="sourceLineNo">3870</span><a name="line.3870"></a>
<span class="sourceLineNo">3871</span>            // Update my commit index<a name="line.3871"></a>
<span class="sourceLineNo">3872</span>            final long newCommitIndex = Math.min(Math.max(leaderCommitIndex, this.raft.commitIndex), lastLogIndex);<a name="line.3872"></a>
<span class="sourceLineNo">3873</span>            if (newCommitIndex &gt; this.raft.commitIndex) {<a name="line.3873"></a>
<span class="sourceLineNo">3874</span>                if (this.log.isDebugEnabled())<a name="line.3874"></a>
<span class="sourceLineNo">3875</span>                    this.debug("updating leader commit index from " + this.raft.commitIndex + " -&gt; " + newCommitIndex);<a name="line.3875"></a>
<span class="sourceLineNo">3876</span>                this.raft.commitIndex = newCommitIndex;<a name="line.3876"></a>
<span class="sourceLineNo">3877</span>                this.raft.requestService(this.checkWaitingTransactionsService);<a name="line.3877"></a>
<span class="sourceLineNo">3878</span>                this.raft.requestService(this.applyCommittedLogEntriesService);<a name="line.3878"></a>
<span class="sourceLineNo">3879</span>            }<a name="line.3879"></a>
<span class="sourceLineNo">3880</span><a name="line.3880"></a>
<span class="sourceLineNo">3881</span>            // Debug<a name="line.3881"></a>
<span class="sourceLineNo">3882</span>            if (this.log.isTraceEnabled()) {<a name="line.3882"></a>
<span class="sourceLineNo">3883</span>                this.trace("my updated follower state: "<a name="line.3883"></a>
<span class="sourceLineNo">3884</span>                  + "term=" + this.raft.currentTerm<a name="line.3884"></a>
<span class="sourceLineNo">3885</span>                  + " commitIndex=" + this.raft.commitIndex<a name="line.3885"></a>
<span class="sourceLineNo">3886</span>                  + " leaderLeaseTimeout=" + this.leaderLeaseTimeout<a name="line.3886"></a>
<span class="sourceLineNo">3887</span>                  + " lastApplied=" + this.raft.lastAppliedTerm + "/" + this.raft.lastAppliedIndex<a name="line.3887"></a>
<span class="sourceLineNo">3888</span>                  + " log=" + this.raft.raftLog);<a name="line.3888"></a>
<span class="sourceLineNo">3889</span>            }<a name="line.3889"></a>
<span class="sourceLineNo">3890</span><a name="line.3890"></a>
<span class="sourceLineNo">3891</span>            // Send reply<a name="line.3891"></a>
<span class="sourceLineNo">3892</span>            if (success) {<a name="line.3892"></a>
<span class="sourceLineNo">3893</span>                this.raft.sendMessage(new AppendResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3893"></a>
<span class="sourceLineNo">3894</span>                  this.raft.currentTerm, msg.getLeaderTimestamp(), true, msg.isProbe() ? logIndex - 1 : logIndex,<a name="line.3894"></a>
<span class="sourceLineNo">3895</span>                  this.raft.getLastLogIndex()));<a name="line.3895"></a>
<span class="sourceLineNo">3896</span>            } else {<a name="line.3896"></a>
<span class="sourceLineNo">3897</span>                this.raft.sendMessage(new AppendResponse(this.raft.clusterId, this.raft.identity, msg.getSenderId(),<a name="line.3897"></a>
<span class="sourceLineNo">3898</span>                  this.raft.currentTerm, msg.getLeaderTimestamp(), false, this.raft.lastAppliedIndex, this.raft.getLastLogIndex()));<a name="line.3898"></a>
<span class="sourceLineNo">3899</span>            }<a name="line.3899"></a>
<span class="sourceLineNo">3900</span>        }<a name="line.3900"></a>
<span class="sourceLineNo">3901</span><a name="line.3901"></a>
<span class="sourceLineNo">3902</span>        @Override<a name="line.3902"></a>
<span class="sourceLineNo">3903</span>        public void caseCommitResponse(CommitResponse msg) {<a name="line.3903"></a>
<span class="sourceLineNo">3904</span><a name="line.3904"></a>
<span class="sourceLineNo">3905</span>            // Find transaction<a name="line.3905"></a>
<span class="sourceLineNo">3906</span>            final RaftKVTransaction tx = this.raft.openTransactions.get(msg.getTxId());<a name="line.3906"></a>
<span class="sourceLineNo">3907</span>            if (tx == null)                                                                 // must have been rolled back locally<a name="line.3907"></a>
<span class="sourceLineNo">3908</span>                return;<a name="line.3908"></a>
<span class="sourceLineNo">3909</span><a name="line.3909"></a>
<span class="sourceLineNo">3910</span>            // Sanity check transaction state<a name="line.3910"></a>
<span class="sourceLineNo">3911</span>            if (!tx.getState().equals(TxState.COMMIT_READY)) {<a name="line.3911"></a>
<span class="sourceLineNo">3912</span>                this.warn("rec'd " + msg + " for " + tx + " in state " + tx.getState() + "; ignoring");<a name="line.3912"></a>
<span class="sourceLineNo">3913</span>                return;<a name="line.3913"></a>
<span class="sourceLineNo">3914</span>            }<a name="line.3914"></a>
<span class="sourceLineNo">3915</span>            if (this.pendingRequests.remove(tx.getTxId()) == null) {<a name="line.3915"></a>
<span class="sourceLineNo">3916</span>                if (this.log.isDebugEnabled())<a name="line.3916"></a>
<span class="sourceLineNo">3917</span>                    this.debug("rec'd " + msg + " for " + tx + " not expecting a response; ignoring");<a name="line.3917"></a>
<span class="sourceLineNo">3918</span>                return;<a name="line.3918"></a>
<span class="sourceLineNo">3919</span>            }<a name="line.3919"></a>
<span class="sourceLineNo">3920</span><a name="line.3920"></a>
<span class="sourceLineNo">3921</span>            // Check result<a name="line.3921"></a>
<span class="sourceLineNo">3922</span>            if (this.log.isTraceEnabled())<a name="line.3922"></a>
<span class="sourceLineNo">3923</span>                this.trace("rec'd " + msg + " for " + tx);<a name="line.3923"></a>
<span class="sourceLineNo">3924</span>            if (msg.isSuccess()) {<a name="line.3924"></a>
<span class="sourceLineNo">3925</span>                tx.setCommitTerm(msg.getCommitTerm());<a name="line.3925"></a>
<span class="sourceLineNo">3926</span>                tx.setCommitIndex(msg.getCommitIndex());<a name="line.3926"></a>
<span class="sourceLineNo">3927</span>                if (msg.getCommitLeaderLeaseTimeout() != null)<a name="line.3927"></a>
<span class="sourceLineNo">3928</span>                    this.commitLeaderLeaseTimeoutMap.put(tx.getTxId(), msg.getCommitLeaderLeaseTimeout());<a name="line.3928"></a>
<span class="sourceLineNo">3929</span>                tx.setState(TxState.COMMIT_WAITING);<a name="line.3929"></a>
<span class="sourceLineNo">3930</span>                tx.getMutableView().disableReadTracking();                                  // we no longer need this info<a name="line.3930"></a>
<span class="sourceLineNo">3931</span>                this.raft.requestService(new CheckWaitingTransactionService(tx));<a name="line.3931"></a>
<span class="sourceLineNo">3932</span>            } else<a name="line.3932"></a>
<span class="sourceLineNo">3933</span>                this.raft.fail(tx, new RetryTransactionException(tx, msg.getErrorMessage()));<a name="line.3933"></a>
<span class="sourceLineNo">3934</span>        }<a name="line.3934"></a>
<span class="sourceLineNo">3935</span><a name="line.3935"></a>
<span class="sourceLineNo">3936</span>        @Override<a name="line.3936"></a>
<span class="sourceLineNo">3937</span>        public void caseInstallSnapshot(InstallSnapshot msg) {<a name="line.3937"></a>
<span class="sourceLineNo">3938</span><a name="line.3938"></a>
<span class="sourceLineNo">3939</span>            // Restart election timer (if running)<a name="line.3939"></a>
<span class="sourceLineNo">3940</span>            if (this.electionTimer.isRunning())<a name="line.3940"></a>
<span class="sourceLineNo">3941</span>                this.restartElectionTimer();<a name="line.3941"></a>
<span class="sourceLineNo">3942</span><a name="line.3942"></a>
<span class="sourceLineNo">3943</span>            // Do we have an existing install?<a name="line.3943"></a>
<span class="sourceLineNo">3944</span>            boolean startNewInstall = false;<a name="line.3944"></a>
<span class="sourceLineNo">3945</span>            if (this.snapshotReceive != null) {<a name="line.3945"></a>
<span class="sourceLineNo">3946</span><a name="line.3946"></a>
<span class="sourceLineNo">3947</span>                // Does the message not match?<a name="line.3947"></a>
<span class="sourceLineNo">3948</span>                if (!this.snapshotReceive.matches(msg)) {<a name="line.3948"></a>
<span class="sourceLineNo">3949</span><a name="line.3949"></a>
<span class="sourceLineNo">3950</span>                    // If the message is NOT the first one in a new install, ignore it<a name="line.3950"></a>
<span class="sourceLineNo">3951</span>                    if (msg.getPairIndex() != 0) {<a name="line.3951"></a>
<span class="sourceLineNo">3952</span>                        if (this.log.isDebugEnabled())<a name="line.3952"></a>
<span class="sourceLineNo">3953</span>                            this.debug("rec'd " + msg + " which doesn't match in-progress " + this.snapshotReceive + "; ignoring");<a name="line.3953"></a>
<span class="sourceLineNo">3954</span>                        return;<a name="line.3954"></a>
<span class="sourceLineNo">3955</span>                    }<a name="line.3955"></a>
<span class="sourceLineNo">3956</span><a name="line.3956"></a>
<span class="sourceLineNo">3957</span>                    // The message is the first one in a new install, so discard the existing install<a name="line.3957"></a>
<span class="sourceLineNo">3958</span>                    this.info("rec'd initial " + msg + " with in-progress " + this.snapshotReceive + "; aborting previous install");<a name="line.3958"></a>
<span class="sourceLineNo">3959</span>                    startNewInstall = true;<a name="line.3959"></a>
<span class="sourceLineNo">3960</span>                }<a name="line.3960"></a>
<span class="sourceLineNo">3961</span>            } else {<a name="line.3961"></a>
<span class="sourceLineNo">3962</span><a name="line.3962"></a>
<span class="sourceLineNo">3963</span>                // If the message is NOT the first one in a new install, ignore it<a name="line.3963"></a>
<span class="sourceLineNo">3964</span>                if (msg.getPairIndex() != 0) {<a name="line.3964"></a>
<span class="sourceLineNo">3965</span>                    this.info("rec'd non-initial " + msg + " with no in-progress snapshot install; ignoring");<a name="line.3965"></a>
<span class="sourceLineNo">3966</span>                    return;<a name="line.3966"></a>
<span class="sourceLineNo">3967</span>                }<a name="line.3967"></a>
<span class="sourceLineNo">3968</span>            }<a name="line.3968"></a>
<span class="sourceLineNo">3969</span><a name="line.3969"></a>
<span class="sourceLineNo">3970</span>            // Get snapshot term and index<a name="line.3970"></a>
<span class="sourceLineNo">3971</span>            final long term = msg.getSnapshotTerm();<a name="line.3971"></a>
<span class="sourceLineNo">3972</span>            final long index = msg.getSnapshotIndex();<a name="line.3972"></a>
<span class="sourceLineNo">3973</span><a name="line.3973"></a>
<span class="sourceLineNo">3974</span>            // Set up new install if necessary<a name="line.3974"></a>
<span class="sourceLineNo">3975</span>            if (this.snapshotReceive == null || startNewInstall) {<a name="line.3975"></a>
<span class="sourceLineNo">3976</span>                assert msg.getPairIndex() == 0;<a name="line.3976"></a>
<span class="sourceLineNo">3977</span>                if (!this.raft.resetStateMachine(false))<a name="line.3977"></a>
<span class="sourceLineNo">3978</span>                    return;<a name="line.3978"></a>
<span class="sourceLineNo">3979</span>                this.updateElectionTimer();<a name="line.3979"></a>
<span class="sourceLineNo">3980</span>                this.snapshotReceive = new SnapshotReceive(<a name="line.3980"></a>
<span class="sourceLineNo">3981</span>                  PrefixKVStore.create(this.raft.kv, STATE_MACHINE_PREFIX), term, index, msg.getSnapshotConfig());<a name="line.3981"></a>
<span class="sourceLineNo">3982</span>                this.info("starting new snapshot install from \"" + msg.getSenderId() + "\" of "<a name="line.3982"></a>
<span class="sourceLineNo">3983</span>                  + term + "/" + index + " with config " + msg.getSnapshotConfig());<a name="line.3983"></a>
<span class="sourceLineNo">3984</span>            }<a name="line.3984"></a>
<span class="sourceLineNo">3985</span>            assert this.snapshotReceive.matches(msg);<a name="line.3985"></a>
<span class="sourceLineNo">3986</span><a name="line.3986"></a>
<span class="sourceLineNo">3987</span>            // Apply next chunk of key/value pairs<a name="line.3987"></a>
<span class="sourceLineNo">3988</span>            if (this.log.isDebugEnabled())<a name="line.3988"></a>
<span class="sourceLineNo">3989</span>                this.debug("applying " + msg + " to " + this.snapshotReceive);<a name="line.3989"></a>
<span class="sourceLineNo">3990</span>            try {<a name="line.3990"></a>
<span class="sourceLineNo">3991</span>                this.snapshotReceive.applyNextChunk(msg.getData());<a name="line.3991"></a>
<span class="sourceLineNo">3992</span>            } catch (Exception e) {<a name="line.3992"></a>
<span class="sourceLineNo">3993</span>                this.error("error applying snapshot to key/value store; resetting state machine", e);<a name="line.3993"></a>
<span class="sourceLineNo">3994</span>                if (!this.raft.resetStateMachine(true))<a name="line.3994"></a>
<span class="sourceLineNo">3995</span>                    return;<a name="line.3995"></a>
<span class="sourceLineNo">3996</span>                this.updateElectionTimer();<a name="line.3996"></a>
<span class="sourceLineNo">3997</span>                this.snapshotReceive = null;<a name="line.3997"></a>
<span class="sourceLineNo">3998</span>                return;<a name="line.3998"></a>
<span class="sourceLineNo">3999</span>            }<a name="line.3999"></a>
<span class="sourceLineNo">4000</span><a name="line.4000"></a>
<span class="sourceLineNo">4001</span>            // If that was the last chunk, finalize persistent state<a name="line.4001"></a>
<span class="sourceLineNo">4002</span>            if (msg.isLastChunk()) {<a name="line.4002"></a>
<span class="sourceLineNo">4003</span>                final Map&lt;String, String&gt; snapshotConfig = this.snapshotReceive.getSnapshotConfig();<a name="line.4003"></a>
<span class="sourceLineNo">4004</span>                this.info("snapshot install from \"" + msg.getSenderId() + "\" of " + term + "/" + index<a name="line.4004"></a>
<span class="sourceLineNo">4005</span>                  + " with config " + snapshotConfig + " complete");<a name="line.4005"></a>
<span class="sourceLineNo">4006</span>                this.snapshotReceive = null;<a name="line.4006"></a>
<span class="sourceLineNo">4007</span>                if (!this.raft.recordLastApplied(term, index, snapshotConfig))<a name="line.4007"></a>
<span class="sourceLineNo">4008</span>                    this.raft.resetStateMachine(true);                          // if this fails we are really screwed<a name="line.4008"></a>
<span class="sourceLineNo">4009</span>                this.updateElectionTimer();<a name="line.4009"></a>
<span class="sourceLineNo">4010</span>            }<a name="line.4010"></a>
<span class="sourceLineNo">4011</span>        }<a name="line.4011"></a>
<span class="sourceLineNo">4012</span><a name="line.4012"></a>
<span class="sourceLineNo">4013</span>        @Override<a name="line.4013"></a>
<span class="sourceLineNo">4014</span>        public void caseRequestVote(RequestVote msg) {<a name="line.4014"></a>
<span class="sourceLineNo">4015</span><a name="line.4015"></a>
<span class="sourceLineNo">4016</span>            // Did we already vote for somebody else?<a name="line.4016"></a>
<span class="sourceLineNo">4017</span>            final String peer = msg.getSenderId();<a name="line.4017"></a>
<span class="sourceLineNo">4018</span>            if (this.votedFor != null &amp;&amp; !this.votedFor.equals(peer)) {<a name="line.4018"></a>
<span class="sourceLineNo">4019</span>                this.info("rec'd " + msg + "; rejected because we already voted for \"" + this.votedFor + "\"");<a name="line.4019"></a>
<span class="sourceLineNo">4020</span>                return;<a name="line.4020"></a>
<span class="sourceLineNo">4021</span>            }<a name="line.4021"></a>
<span class="sourceLineNo">4022</span><a name="line.4022"></a>
<span class="sourceLineNo">4023</span>            // Verify that we are allowed to vote for this peer<a name="line.4023"></a>
<span class="sourceLineNo">4024</span>            if (msg.getLastLogTerm() &lt; this.raft.getLastLogTerm()<a name="line.4024"></a>
<span class="sourceLineNo">4025</span>              || (msg.getLastLogTerm() == this.raft.getLastLogTerm() &amp;&amp; msg.getLastLogIndex() &lt; this.raft.getLastLogIndex())) {<a name="line.4025"></a>
<span class="sourceLineNo">4026</span>                this.info("rec'd " + msg + "; rejected because their log " + msg.getLastLogTerm() + "/"<a name="line.4026"></a>
<span class="sourceLineNo">4027</span>                  + msg.getLastLogIndex() + " loses to ours " + this.raft.getLastLogTerm() + "/" + this.raft.getLastLogIndex());<a name="line.4027"></a>
<span class="sourceLineNo">4028</span>                return;<a name="line.4028"></a>
<span class="sourceLineNo">4029</span>            }<a name="line.4029"></a>
<span class="sourceLineNo">4030</span><a name="line.4030"></a>
<span class="sourceLineNo">4031</span>            // Persist our vote for this peer (if not already persisted)<a name="line.4031"></a>
<span class="sourceLineNo">4032</span>            if (this.votedFor == null) {<a name="line.4032"></a>
<span class="sourceLineNo">4033</span>                this.info("granting vote to \"" + peer + "\" in term " + this.raft.currentTerm);<a name="line.4033"></a>
<span class="sourceLineNo">4034</span>                if (!this.updateVotedFor(peer))<a name="line.4034"></a>
<span class="sourceLineNo">4035</span>                    return;<a name="line.4035"></a>
<span class="sourceLineNo">4036</span>            } else<a name="line.4036"></a>
<span class="sourceLineNo">4037</span>                this.info("confirming existing vote for \"" + peer + "\" in term " + this.raft.currentTerm);<a name="line.4037"></a>
<span class="sourceLineNo">4038</span><a name="line.4038"></a>
<span class="sourceLineNo">4039</span>            // Send reply<a name="line.4039"></a>
<span class="sourceLineNo">4040</span>            this.raft.sendMessage(new GrantVote(this.raft.clusterId, this.raft.identity, peer, this.raft.currentTerm));<a name="line.4040"></a>
<span class="sourceLineNo">4041</span>        }<a name="line.4041"></a>
<span class="sourceLineNo">4042</span><a name="line.4042"></a>
<span class="sourceLineNo">4043</span>        @Override<a name="line.4043"></a>
<span class="sourceLineNo">4044</span>        public void caseGrantVote(GrantVote msg) {<a name="line.4044"></a>
<span class="sourceLineNo">4045</span><a name="line.4045"></a>
<span class="sourceLineNo">4046</span>            // Ignore - we already lost the election to the real leader<a name="line.4046"></a>
<span class="sourceLineNo">4047</span>            if (this.log.isDebugEnabled())<a name="line.4047"></a>
<span class="sourceLineNo">4048</span>                this.debug("ignoring " + msg + " rec'd while in " + this);<a name="line.4048"></a>
<span class="sourceLineNo">4049</span>        }<a name="line.4049"></a>
<span class="sourceLineNo">4050</span><a name="line.4050"></a>
<span class="sourceLineNo">4051</span>    // Helper methods<a name="line.4051"></a>
<span class="sourceLineNo">4052</span><a name="line.4052"></a>
<span class="sourceLineNo">4053</span>        /**<a name="line.4053"></a>
<span class="sourceLineNo">4054</span>         * Record the peer voted for in the current term.<a name="line.4054"></a>
<span class="sourceLineNo">4055</span>         */<a name="line.4055"></a>
<span class="sourceLineNo">4056</span>        private boolean updateVotedFor(String recipient) {<a name="line.4056"></a>
<span class="sourceLineNo">4057</span><a name="line.4057"></a>
<span class="sourceLineNo">4058</span>            // Sanity check<a name="line.4058"></a>
<span class="sourceLineNo">4059</span>            assert Thread.holdsLock(this.raft);<a name="line.4059"></a>
<span class="sourceLineNo">4060</span>            assert recipient != null;<a name="line.4060"></a>
<span class="sourceLineNo">4061</span><a name="line.4061"></a>
<span class="sourceLineNo">4062</span>            // Update persistent store<a name="line.4062"></a>
<span class="sourceLineNo">4063</span>            final Writes writes = new Writes();<a name="line.4063"></a>
<span class="sourceLineNo">4064</span>            writes.getPuts().put(VOTED_FOR_KEY, this.raft.encodeString(recipient));<a name="line.4064"></a>
<span class="sourceLineNo">4065</span>            try {<a name="line.4065"></a>
<span class="sourceLineNo">4066</span>                this.raft.kv.mutate(writes, true);<a name="line.4066"></a>
<span class="sourceLineNo">4067</span>            } catch (Exception e) {<a name="line.4067"></a>
<span class="sourceLineNo">4068</span>                this.error("error persisting vote for \"" + recipient + "\"", e);<a name="line.4068"></a>
<span class="sourceLineNo">4069</span>                return false;<a name="line.4069"></a>
<span class="sourceLineNo">4070</span>            }<a name="line.4070"></a>
<span class="sourceLineNo">4071</span><a name="line.4071"></a>
<span class="sourceLineNo">4072</span>            // Done<a name="line.4072"></a>
<span class="sourceLineNo">4073</span>            this.votedFor = recipient;<a name="line.4073"></a>
<span class="sourceLineNo">4074</span>            return true;<a name="line.4074"></a>
<span class="sourceLineNo">4075</span>        }<a name="line.4075"></a>
<span class="sourceLineNo">4076</span><a name="line.4076"></a>
<span class="sourceLineNo">4077</span>    // Object<a name="line.4077"></a>
<span class="sourceLineNo">4078</span><a name="line.4078"></a>
<span class="sourceLineNo">4079</span>        @Override<a name="line.4079"></a>
<span class="sourceLineNo">4080</span>        public String toString() {<a name="line.4080"></a>
<span class="sourceLineNo">4081</span>            return this.toStringPrefix()<a name="line.4081"></a>
<span class="sourceLineNo">4082</span>              + (this.leader != null ? ",leader=\"" + this.leader + "\"" : "")<a name="line.4082"></a>
<span class="sourceLineNo">4083</span>              + (this.votedFor != null ? ",votedFor=\"" + this.votedFor + "\"" : "")<a name="line.4083"></a>
<span class="sourceLineNo">4084</span>              + (!this.pendingRequests.isEmpty() ? ",pendingRequests=" + this.pendingRequests.keySet() : "")<a name="line.4084"></a>
<span class="sourceLineNo">4085</span>              + (!this.pendingWrites.isEmpty() ? ",pendingWrites=" + this.pendingWrites.keySet() : "")<a name="line.4085"></a>
<span class="sourceLineNo">4086</span>              + (!this.commitLeaderLeaseTimeoutMap.isEmpty() ? ",leaseTimeouts=" + this.commitLeaderLeaseTimeoutMap.keySet() : "")<a name="line.4086"></a>
<span class="sourceLineNo">4087</span>              + "]";<a name="line.4087"></a>
<span class="sourceLineNo">4088</span>        }<a name="line.4088"></a>
<span class="sourceLineNo">4089</span><a name="line.4089"></a>
<span class="sourceLineNo">4090</span>    // Debug<a name="line.4090"></a>
<span class="sourceLineNo">4091</span><a name="line.4091"></a>
<span class="sourceLineNo">4092</span>        @Override<a name="line.4092"></a>
<span class="sourceLineNo">4093</span>        boolean checkState() {<a name="line.4093"></a>
<span class="sourceLineNo">4094</span>            if (!super.checkState())<a name="line.4094"></a>
<span class="sourceLineNo">4095</span>                return false;<a name="line.4095"></a>
<span class="sourceLineNo">4096</span>            assert this.leaderAddress != null || this.leader == null;<a name="line.4096"></a>
<span class="sourceLineNo">4097</span>            assert this.electionTimer.isRunning() == this.raft.isClusterMember();<a name="line.4097"></a>
<span class="sourceLineNo">4098</span>            for (Map.Entry&lt;Long, PendingRequest&gt; entry : this.pendingRequests.entrySet()) {<a name="line.4098"></a>
<span class="sourceLineNo">4099</span>                final long txId = entry.getKey();<a name="line.4099"></a>
<span class="sourceLineNo">4100</span>                final PendingRequest pendingRequest = entry.getValue();<a name="line.4100"></a>
<span class="sourceLineNo">4101</span>                final RaftKVTransaction tx = pendingRequest.getTx();<a name="line.4101"></a>
<span class="sourceLineNo">4102</span>                assert txId == tx.getTxId();<a name="line.4102"></a>
<span class="sourceLineNo">4103</span>                assert tx.getState().equals(TxState.COMMIT_READY);<a name="line.4103"></a>
<span class="sourceLineNo">4104</span>                assert tx.getCommitTerm() == 0;<a name="line.4104"></a>
<span class="sourceLineNo">4105</span>                assert tx.getCommitIndex() == 0;<a name="line.4105"></a>
<span class="sourceLineNo">4106</span>            }<a name="line.4106"></a>
<span class="sourceLineNo">4107</span>            for (Map.Entry&lt;Long, PendingWrite&gt; entry : this.pendingWrites.entrySet()) {<a name="line.4107"></a>
<span class="sourceLineNo">4108</span>                final long txId = entry.getKey();<a name="line.4108"></a>
<span class="sourceLineNo">4109</span>                final PendingWrite pendingWrite = entry.getValue();<a name="line.4109"></a>
<span class="sourceLineNo">4110</span>                final RaftKVTransaction tx = pendingWrite.getTx();<a name="line.4110"></a>
<span class="sourceLineNo">4111</span>                assert txId == tx.getTxId();<a name="line.4111"></a>
<span class="sourceLineNo">4112</span>                assert tx.getState().equals(TxState.COMMIT_READY) || tx.getState().equals(TxState.COMMIT_WAITING);<a name="line.4112"></a>
<span class="sourceLineNo">4113</span>                assert pendingWrite.getFileWriter().getFile().exists();<a name="line.4113"></a>
<span class="sourceLineNo">4114</span>            }<a name="line.4114"></a>
<span class="sourceLineNo">4115</span>            return true;<a name="line.4115"></a>
<span class="sourceLineNo">4116</span>        }<a name="line.4116"></a>
<span class="sourceLineNo">4117</span><a name="line.4117"></a>
<span class="sourceLineNo">4118</span>    // PendingRequest<a name="line.4118"></a>
<span class="sourceLineNo">4119</span><a name="line.4119"></a>
<span class="sourceLineNo">4120</span>        // Represents a transaction in COMMIT_READY for which a CommitRequest has been sent to the leader<a name="line.4120"></a>
<span class="sourceLineNo">4121</span>        // but no CommitResponse has yet been received<a name="line.4121"></a>
<span class="sourceLineNo">4122</span>        private class PendingRequest {<a name="line.4122"></a>
<span class="sourceLineNo">4123</span><a name="line.4123"></a>
<span class="sourceLineNo">4124</span>            private final RaftKVTransaction tx;<a name="line.4124"></a>
<span class="sourceLineNo">4125</span><a name="line.4125"></a>
<span class="sourceLineNo">4126</span>            PendingRequest(RaftKVTransaction tx) {<a name="line.4126"></a>
<span class="sourceLineNo">4127</span>                this.tx = tx;<a name="line.4127"></a>
<span class="sourceLineNo">4128</span>                assert !FollowerRole.this.pendingRequests.containsKey(tx.getTxId());<a name="line.4128"></a>
<span class="sourceLineNo">4129</span>                FollowerRole.this.pendingRequests.put(tx.getTxId(), this);<a name="line.4129"></a>
<span class="sourceLineNo">4130</span>            }<a name="line.4130"></a>
<span class="sourceLineNo">4131</span><a name="line.4131"></a>
<span class="sourceLineNo">4132</span>            public RaftKVTransaction getTx() {<a name="line.4132"></a>
<span class="sourceLineNo">4133</span>                return this.tx;<a name="line.4133"></a>
<span class="sourceLineNo">4134</span>            }<a name="line.4134"></a>
<span class="sourceLineNo">4135</span>        }<a name="line.4135"></a>
<span class="sourceLineNo">4136</span><a name="line.4136"></a>
<span class="sourceLineNo">4137</span>    // PendingWrite<a name="line.4137"></a>
<span class="sourceLineNo">4138</span><a name="line.4138"></a>
<span class="sourceLineNo">4139</span>        // Represents a read-write transaction in COMMIT_READY or COMMIT_WAITING for which the server's AppendRequest<a name="line.4139"></a>
<span class="sourceLineNo">4140</span>        // will have null mutationData, because we will already have the data on hand waiting in a temporary file. This<a name="line.4140"></a>
<span class="sourceLineNo">4141</span>        // is a simple optimization to avoid sending the same data from leader -&gt; follower just sent from follower -&gt; leader.<a name="line.4141"></a>
<span class="sourceLineNo">4142</span>        private class PendingWrite {<a name="line.4142"></a>
<span class="sourceLineNo">4143</span><a name="line.4143"></a>
<span class="sourceLineNo">4144</span>            private final RaftKVTransaction tx;<a name="line.4144"></a>
<span class="sourceLineNo">4145</span>            private final FileWriter fileWriter;<a name="line.4145"></a>
<span class="sourceLineNo">4146</span><a name="line.4146"></a>
<span class="sourceLineNo">4147</span>            PendingWrite(RaftKVTransaction tx, FileWriter fileWriter) {<a name="line.4147"></a>
<span class="sourceLineNo">4148</span>                this.tx = tx;<a name="line.4148"></a>
<span class="sourceLineNo">4149</span>                this.fileWriter = fileWriter;<a name="line.4149"></a>
<span class="sourceLineNo">4150</span>            }<a name="line.4150"></a>
<span class="sourceLineNo">4151</span><a name="line.4151"></a>
<span class="sourceLineNo">4152</span>            public RaftKVTransaction getTx() {<a name="line.4152"></a>
<span class="sourceLineNo">4153</span>                return this.tx;<a name="line.4153"></a>
<span class="sourceLineNo">4154</span>            }<a name="line.4154"></a>
<span class="sourceLineNo">4155</span><a name="line.4155"></a>
<span class="sourceLineNo">4156</span>            public FileWriter getFileWriter() {<a name="line.4156"></a>
<span class="sourceLineNo">4157</span>                return this.fileWriter;<a name="line.4157"></a>
<span class="sourceLineNo">4158</span>            }<a name="line.4158"></a>
<span class="sourceLineNo">4159</span><a name="line.4159"></a>
<span class="sourceLineNo">4160</span>            public void cleanup() {<a name="line.4160"></a>
<span class="sourceLineNo">4161</span>                this.fileWriter.getFile().delete();<a name="line.4161"></a>
<span class="sourceLineNo">4162</span>                Util.closeIfPossible(this.fileWriter);<a name="line.4162"></a>
<span class="sourceLineNo">4163</span>            }<a name="line.4163"></a>
<span class="sourceLineNo">4164</span>        }<a name="line.4164"></a>
<span class="sourceLineNo">4165</span>    }<a name="line.4165"></a>
<span class="sourceLineNo">4166</span><a name="line.4166"></a>
<span class="sourceLineNo">4167</span>// CANDIDATE role<a name="line.4167"></a>
<span class="sourceLineNo">4168</span><a name="line.4168"></a>
<span class="sourceLineNo">4169</span>    private static class CandidateRole extends NonLeaderRole {<a name="line.4169"></a>
<span class="sourceLineNo">4170</span><a name="line.4170"></a>
<span class="sourceLineNo">4171</span>        private final HashSet&lt;String&gt; votes = new HashSet&lt;&gt;();<a name="line.4171"></a>
<span class="sourceLineNo">4172</span><a name="line.4172"></a>
<span class="sourceLineNo">4173</span>    // Constructors<a name="line.4173"></a>
<span class="sourceLineNo">4174</span><a name="line.4174"></a>
<span class="sourceLineNo">4175</span>        public CandidateRole(RaftKVDatabase raft) {<a name="line.4175"></a>
<span class="sourceLineNo">4176</span>            super(raft, true);<a name="line.4176"></a>
<span class="sourceLineNo">4177</span>        }<a name="line.4177"></a>
<span class="sourceLineNo">4178</span><a name="line.4178"></a>
<span class="sourceLineNo">4179</span>    // Lifecycle<a name="line.4179"></a>
<span class="sourceLineNo">4180</span><a name="line.4180"></a>
<span class="sourceLineNo">4181</span>        @Override<a name="line.4181"></a>
<span class="sourceLineNo">4182</span>        public void setup() {<a name="line.4182"></a>
<span class="sourceLineNo">4183</span>            super.setup();<a name="line.4183"></a>
<span class="sourceLineNo">4184</span><a name="line.4184"></a>
<span class="sourceLineNo">4185</span>            // Increment term<a name="line.4185"></a>
<span class="sourceLineNo">4186</span>            if (!this.raft.advanceTerm(this.raft.currentTerm + 1))<a name="line.4186"></a>
<span class="sourceLineNo">4187</span>                return;<a name="line.4187"></a>
<span class="sourceLineNo">4188</span><a name="line.4188"></a>
<span class="sourceLineNo">4189</span>            // Request votes from other peers<a name="line.4189"></a>
<span class="sourceLineNo">4190</span>            final HashSet&lt;String&gt; voters = new HashSet&lt;&gt;(this.raft.currentConfig.keySet());<a name="line.4190"></a>
<span class="sourceLineNo">4191</span>            voters.remove(this.raft.identity);<a name="line.4191"></a>
<span class="sourceLineNo">4192</span>            this.info("entering candidate role in term " + this.raft.currentTerm + "; requesting votes from " + voters);<a name="line.4192"></a>
<span class="sourceLineNo">4193</span>            for (String voter : voters) {<a name="line.4193"></a>
<span class="sourceLineNo">4194</span>                this.raft.sendMessage(new RequestVote(this.raft.clusterId, this.raft.identity, voter,<a name="line.4194"></a>
<span class="sourceLineNo">4195</span>                  this.raft.currentTerm, this.raft.getLastLogTerm(), this.raft.getLastLogIndex()));<a name="line.4195"></a>
<span class="sourceLineNo">4196</span>            }<a name="line.4196"></a>
<span class="sourceLineNo">4197</span>        }<a name="line.4197"></a>
<span class="sourceLineNo">4198</span><a name="line.4198"></a>
<span class="sourceLineNo">4199</span>    // Service<a name="line.4199"></a>
<span class="sourceLineNo">4200</span><a name="line.4200"></a>
<span class="sourceLineNo">4201</span>        @Override<a name="line.4201"></a>
<span class="sourceLineNo">4202</span>        public void outputQueueEmpty(String address) {<a name="line.4202"></a>
<span class="sourceLineNo">4203</span>            // nothing to do<a name="line.4203"></a>
<span class="sourceLineNo">4204</span>        }<a name="line.4204"></a>
<span class="sourceLineNo">4205</span><a name="line.4205"></a>
<span class="sourceLineNo">4206</span>    // Transactions<a name="line.4206"></a>
<span class="sourceLineNo">4207</span><a name="line.4207"></a>
<span class="sourceLineNo">4208</span>        @Override<a name="line.4208"></a>
<span class="sourceLineNo">4209</span>        public void checkReadyTransaction(RaftKVTransaction tx) {<a name="line.4209"></a>
<span class="sourceLineNo">4210</span><a name="line.4210"></a>
<span class="sourceLineNo">4211</span>            // Sanity check<a name="line.4211"></a>
<span class="sourceLineNo">4212</span>            assert Thread.holdsLock(this.raft);<a name="line.4212"></a>
<span class="sourceLineNo">4213</span>            assert tx.getState().equals(TxState.COMMIT_READY);<a name="line.4213"></a>
<span class="sourceLineNo">4214</span><a name="line.4214"></a>
<span class="sourceLineNo">4215</span>            // Check snapshot isolation<a name="line.4215"></a>
<span class="sourceLineNo">4216</span>            if (this.checkReadyTransactionReadOnlySnapshot(tx))<a name="line.4216"></a>
<span class="sourceLineNo">4217</span>                return;<a name="line.4217"></a>
<span class="sourceLineNo">4218</span><a name="line.4218"></a>
<span class="sourceLineNo">4219</span>            // We can't do anything with it until we have a leader<a name="line.4219"></a>
<span class="sourceLineNo">4220</span>        }<a name="line.4220"></a>
<span class="sourceLineNo">4221</span><a name="line.4221"></a>
<span class="sourceLineNo">4222</span>        @Override<a name="line.4222"></a>
<span class="sourceLineNo">4223</span>        public void cleanupForTransaction(RaftKVTransaction tx) {<a name="line.4223"></a>
<span class="sourceLineNo">4224</span>            // nothing to do<a name="line.4224"></a>
<span class="sourceLineNo">4225</span>        }<a name="line.4225"></a>
<span class="sourceLineNo">4226</span><a name="line.4226"></a>
<span class="sourceLineNo">4227</span>    // MessageSwitch<a name="line.4227"></a>
<span class="sourceLineNo">4228</span><a name="line.4228"></a>
<span class="sourceLineNo">4229</span>        @Override<a name="line.4229"></a>
<span class="sourceLineNo">4230</span>        public void caseAppendRequest(AppendRequest msg) {<a name="line.4230"></a>
<span class="sourceLineNo">4231</span>            this.info("rec'd " + msg + " in " + this + "; reverting to follower");<a name="line.4231"></a>
<span class="sourceLineNo">4232</span>            this.raft.changeRole(new FollowerRole(this.raft, msg.getSenderId(), this.raft.returnAddress));<a name="line.4232"></a>
<span class="sourceLineNo">4233</span>            this.raft.receiveMessage(this.raft.returnAddress, msg);<a name="line.4233"></a>
<span class="sourceLineNo">4234</span>        }<a name="line.4234"></a>
<span class="sourceLineNo">4235</span><a name="line.4235"></a>
<span class="sourceLineNo">4236</span>    // MessageSwitch<a name="line.4236"></a>
<span class="sourceLineNo">4237</span><a name="line.4237"></a>
<span class="sourceLineNo">4238</span>        @Override<a name="line.4238"></a>
<span class="sourceLineNo">4239</span>        public void caseCommitResponse(CommitResponse msg) {<a name="line.4239"></a>
<span class="sourceLineNo">4240</span><a name="line.4240"></a>
<span class="sourceLineNo">4241</span>            // We could not have ever sent a CommitRequest in this term<a name="line.4241"></a>
<span class="sourceLineNo">4242</span>            this.failUnexpectedMessage(msg);<a name="line.4242"></a>
<span class="sourceLineNo">4243</span>        }<a name="line.4243"></a>
<span class="sourceLineNo">4244</span><a name="line.4244"></a>
<span class="sourceLineNo">4245</span>        @Override<a name="line.4245"></a>
<span class="sourceLineNo">4246</span>        public void caseInstallSnapshot(InstallSnapshot msg) {<a name="line.4246"></a>
<span class="sourceLineNo">4247</span><a name="line.4247"></a>
<span class="sourceLineNo">4248</span>            // We could not have ever sent an AppendResponse in this term<a name="line.4248"></a>
<span class="sourceLineNo">4249</span>            this.failUnexpectedMessage(msg);<a name="line.4249"></a>
<span class="sourceLineNo">4250</span>        }<a name="line.4250"></a>
<span class="sourceLineNo">4251</span><a name="line.4251"></a>
<span class="sourceLineNo">4252</span>        @Override<a name="line.4252"></a>
<span class="sourceLineNo">4253</span>        public void caseRequestVote(RequestVote msg) {<a name="line.4253"></a>
<span class="sourceLineNo">4254</span><a name="line.4254"></a>
<span class="sourceLineNo">4255</span>            // Ignore - we are also a candidate and have already voted for ourself<a name="line.4255"></a>
<span class="sourceLineNo">4256</span>            if (this.log.isDebugEnabled())<a name="line.4256"></a>
<span class="sourceLineNo">4257</span>                this.debug("ignoring " + msg + " rec'd while in " + this);<a name="line.4257"></a>
<span class="sourceLineNo">4258</span>        }<a name="line.4258"></a>
<span class="sourceLineNo">4259</span><a name="line.4259"></a>
<span class="sourceLineNo">4260</span>        @Override<a name="line.4260"></a>
<span class="sourceLineNo">4261</span>        public void caseGrantVote(GrantVote msg) {<a name="line.4261"></a>
<span class="sourceLineNo">4262</span><a name="line.4262"></a>
<span class="sourceLineNo">4263</span>            // Record vote<a name="line.4263"></a>
<span class="sourceLineNo">4264</span>            this.votes.add(msg.getSenderId());<a name="line.4264"></a>
<span class="sourceLineNo">4265</span>            this.info("rec'd election vote from \"" + msg.getSenderId() + "\" in term " + this.raft.currentTerm);<a name="line.4265"></a>
<span class="sourceLineNo">4266</span><a name="line.4266"></a>
<span class="sourceLineNo">4267</span>            // Tally votes<a name="line.4267"></a>
<span class="sourceLineNo">4268</span>            final int allVotes = this.raft.currentConfig.size();<a name="line.4268"></a>
<span class="sourceLineNo">4269</span>            final int numVotes = this.votes.size() + (this.raft.isClusterMember() ? 1 : 0);<a name="line.4269"></a>
<span class="sourceLineNo">4270</span>            final int minVotes = allVotes / 2 + 1;                              // require a majority<a name="line.4270"></a>
<span class="sourceLineNo">4271</span><a name="line.4271"></a>
<span class="sourceLineNo">4272</span>            // Did we win?<a name="line.4272"></a>
<span class="sourceLineNo">4273</span>            if (numVotes &gt;= minVotes) {<a name="line.4273"></a>
<span class="sourceLineNo">4274</span>                this.info("won the election for term " + this.raft.currentTerm + " with "<a name="line.4274"></a>
<span class="sourceLineNo">4275</span>                  + numVotes + "/" + allVotes + " votes; BECOMING LEADER IN TERM " + this.raft.currentTerm);<a name="line.4275"></a>
<span class="sourceLineNo">4276</span>                this.raft.changeRole(new LeaderRole(this.raft));<a name="line.4276"></a>
<span class="sourceLineNo">4277</span>            }<a name="line.4277"></a>
<span class="sourceLineNo">4278</span>        }<a name="line.4278"></a>
<span class="sourceLineNo">4279</span><a name="line.4279"></a>
<span class="sourceLineNo">4280</span>    // Object<a name="line.4280"></a>
<span class="sourceLineNo">4281</span><a name="line.4281"></a>
<span class="sourceLineNo">4282</span>        @Override<a name="line.4282"></a>
<span class="sourceLineNo">4283</span>        public String toString() {<a name="line.4283"></a>
<span class="sourceLineNo">4284</span>            return this.toStringPrefix()<a name="line.4284"></a>
<span class="sourceLineNo">4285</span>              + ",votes=" + this.votes<a name="line.4285"></a>
<span class="sourceLineNo">4286</span>              + "]";<a name="line.4286"></a>
<span class="sourceLineNo">4287</span>        }<a name="line.4287"></a>
<span class="sourceLineNo">4288</span><a name="line.4288"></a>
<span class="sourceLineNo">4289</span>    // Debug<a name="line.4289"></a>
<span class="sourceLineNo">4290</span><a name="line.4290"></a>
<span class="sourceLineNo">4291</span>        @Override<a name="line.4291"></a>
<span class="sourceLineNo">4292</span>        boolean checkState() {<a name="line.4292"></a>
<span class="sourceLineNo">4293</span>            if (!super.checkState())<a name="line.4293"></a>
<span class="sourceLineNo">4294</span>                return false;<a name="line.4294"></a>
<span class="sourceLineNo">4295</span>            assert this.electionTimer.isRunning();<a name="line.4295"></a>
<span class="sourceLineNo">4296</span>            assert this.raft.isClusterMember();<a name="line.4296"></a>
<span class="sourceLineNo">4297</span>            return true;<a name="line.4297"></a>
<span class="sourceLineNo">4298</span>        }<a name="line.4298"></a>
<span class="sourceLineNo">4299</span>    }<a name="line.4299"></a>
<span class="sourceLineNo">4300</span><a name="line.4300"></a>
<span class="sourceLineNo">4301</span>// Prefix Functions<a name="line.4301"></a>
<span class="sourceLineNo">4302</span><a name="line.4302"></a>
<span class="sourceLineNo">4303</span>    private abstract static class AbstractPrefixFunction&lt;F, T&gt; implements Function&lt;F, T&gt; {<a name="line.4303"></a>
<span class="sourceLineNo">4304</span><a name="line.4304"></a>
<span class="sourceLineNo">4305</span>        protected final byte[] prefix;<a name="line.4305"></a>
<span class="sourceLineNo">4306</span><a name="line.4306"></a>
<span class="sourceLineNo">4307</span>        AbstractPrefixFunction(byte[] prefix) {<a name="line.4307"></a>
<span class="sourceLineNo">4308</span>            this.prefix = prefix;<a name="line.4308"></a>
<span class="sourceLineNo">4309</span>        }<a name="line.4309"></a>
<span class="sourceLineNo">4310</span>    }<a name="line.4310"></a>
<span class="sourceLineNo">4311</span><a name="line.4311"></a>
<span class="sourceLineNo">4312</span>    private static class PrefixKeyRangeFunction extends AbstractPrefixFunction&lt;KeyRange, KeyRange&gt; {<a name="line.4312"></a>
<span class="sourceLineNo">4313</span><a name="line.4313"></a>
<span class="sourceLineNo">4314</span>        PrefixKeyRangeFunction(byte[] prefix) {<a name="line.4314"></a>
<span class="sourceLineNo">4315</span>            super(prefix);<a name="line.4315"></a>
<span class="sourceLineNo">4316</span>        }<a name="line.4316"></a>
<span class="sourceLineNo">4317</span><a name="line.4317"></a>
<span class="sourceLineNo">4318</span>        @Override<a name="line.4318"></a>
<span class="sourceLineNo">4319</span>        public KeyRange apply(KeyRange range) {<a name="line.4319"></a>
<span class="sourceLineNo">4320</span>            return range.prefixedBy(this.prefix);<a name="line.4320"></a>
<span class="sourceLineNo">4321</span>        }<a name="line.4321"></a>
<span class="sourceLineNo">4322</span>    }<a name="line.4322"></a>
<span class="sourceLineNo">4323</span><a name="line.4323"></a>
<span class="sourceLineNo">4324</span>    private static class PrefixPutFunction extends AbstractPrefixFunction&lt;Map.Entry&lt;byte[], byte[]&gt;, Map.Entry&lt;byte[], byte[]&gt;&gt; {<a name="line.4324"></a>
<span class="sourceLineNo">4325</span><a name="line.4325"></a>
<span class="sourceLineNo">4326</span>        PrefixPutFunction(byte[] prefix) {<a name="line.4326"></a>
<span class="sourceLineNo">4327</span>            super(prefix);<a name="line.4327"></a>
<span class="sourceLineNo">4328</span>        }<a name="line.4328"></a>
<span class="sourceLineNo">4329</span><a name="line.4329"></a>
<span class="sourceLineNo">4330</span>        @Override<a name="line.4330"></a>
<span class="sourceLineNo">4331</span>        public Map.Entry&lt;byte[], byte[]&gt; apply(Map.Entry&lt;byte[], byte[]&gt; entry) {<a name="line.4331"></a>
<span class="sourceLineNo">4332</span>            return new AbstractMap.SimpleEntry&lt;byte[], byte[]&gt;(Bytes.concat(this.prefix, entry.getKey()), entry.getValue());<a name="line.4332"></a>
<span class="sourceLineNo">4333</span>        }<a name="line.4333"></a>
<span class="sourceLineNo">4334</span>    }<a name="line.4334"></a>
<span class="sourceLineNo">4335</span><a name="line.4335"></a>
<span class="sourceLineNo">4336</span>    private static class PrefixAdjustFunction extends AbstractPrefixFunction&lt;Map.Entry&lt;byte[], Long&gt;, Map.Entry&lt;byte[], Long&gt;&gt; {<a name="line.4336"></a>
<span class="sourceLineNo">4337</span><a name="line.4337"></a>
<span class="sourceLineNo">4338</span>        PrefixAdjustFunction(byte[] prefix) {<a name="line.4338"></a>
<span class="sourceLineNo">4339</span>            super(prefix);<a name="line.4339"></a>
<span class="sourceLineNo">4340</span>        }<a name="line.4340"></a>
<span class="sourceLineNo">4341</span><a name="line.4341"></a>
<span class="sourceLineNo">4342</span>        @Override<a name="line.4342"></a>
<span class="sourceLineNo">4343</span>        public Map.Entry&lt;byte[], Long&gt; apply(Map.Entry&lt;byte[], Long&gt; entry) {<a name="line.4343"></a>
<span class="sourceLineNo">4344</span>            return new AbstractMap.SimpleEntry&lt;byte[], Long&gt;(Bytes.concat(this.prefix, entry.getKey()), entry.getValue());<a name="line.4344"></a>
<span class="sourceLineNo">4345</span>        }<a name="line.4345"></a>
<span class="sourceLineNo">4346</span>    }<a name="line.4346"></a>
<span class="sourceLineNo">4347</span><a name="line.4347"></a>
<span class="sourceLineNo">4348</span>// Debug/Sanity Checking<a name="line.4348"></a>
<span class="sourceLineNo">4349</span><a name="line.4349"></a>
<span class="sourceLineNo">4350</span>    private abstract class ErrorLoggingRunnable implements Runnable {<a name="line.4350"></a>
<span class="sourceLineNo">4351</span><a name="line.4351"></a>
<span class="sourceLineNo">4352</span>        @Override<a name="line.4352"></a>
<span class="sourceLineNo">4353</span>        public final void run() {<a name="line.4353"></a>
<span class="sourceLineNo">4354</span>            try {<a name="line.4354"></a>
<span class="sourceLineNo">4355</span>                this.doRun();<a name="line.4355"></a>
<span class="sourceLineNo">4356</span>            } catch (Throwable t) {<a name="line.4356"></a>
<span class="sourceLineNo">4357</span>                RaftKVDatabase.this.error("exception in callback", t);<a name="line.4357"></a>
<span class="sourceLineNo">4358</span>            }<a name="line.4358"></a>
<span class="sourceLineNo">4359</span>        }<a name="line.4359"></a>
<span class="sourceLineNo">4360</span><a name="line.4360"></a>
<span class="sourceLineNo">4361</span>        protected abstract void doRun();<a name="line.4361"></a>
<span class="sourceLineNo">4362</span>    }<a name="line.4362"></a>
<span class="sourceLineNo">4363</span><a name="line.4363"></a>
<span class="sourceLineNo">4364</span>    private boolean checkState() {<a name="line.4364"></a>
<span class="sourceLineNo">4365</span>        try {<a name="line.4365"></a>
<span class="sourceLineNo">4366</span>            this.doCheckState();<a name="line.4366"></a>
<span class="sourceLineNo">4367</span>        } catch (AssertionError e) {<a name="line.4367"></a>
<span class="sourceLineNo">4368</span>            throw new AssertionError("checkState() failure for \"" + this.identity + "\"", e);<a name="line.4368"></a>
<span class="sourceLineNo">4369</span>        }<a name="line.4369"></a>
<span class="sourceLineNo">4370</span>        return true;<a name="line.4370"></a>
<span class="sourceLineNo">4371</span>    }<a name="line.4371"></a>
<span class="sourceLineNo">4372</span><a name="line.4372"></a>
<span class="sourceLineNo">4373</span>    private void doCheckState() {<a name="line.4373"></a>
<span class="sourceLineNo">4374</span>        assert Thread.holdsLock(this);<a name="line.4374"></a>
<span class="sourceLineNo">4375</span><a name="line.4375"></a>
<span class="sourceLineNo">4376</span>        // Handle stopped state<a name="line.4376"></a>
<span class="sourceLineNo">4377</span>        if (this.role == null) {<a name="line.4377"></a>
<span class="sourceLineNo">4378</span>            assert this.kv == null;<a name="line.4378"></a>
<span class="sourceLineNo">4379</span>            assert this.random == null;<a name="line.4379"></a>
<span class="sourceLineNo">4380</span>            assert this.currentTerm == 0;<a name="line.4380"></a>
<span class="sourceLineNo">4381</span>            assert this.commitIndex == 0;<a name="line.4381"></a>
<span class="sourceLineNo">4382</span>            assert this.lastAppliedTerm == 0;<a name="line.4382"></a>
<span class="sourceLineNo">4383</span>            assert this.lastAppliedIndex == 0;<a name="line.4383"></a>
<span class="sourceLineNo">4384</span>            assert this.lastAppliedConfig == null;<a name="line.4384"></a>
<span class="sourceLineNo">4385</span>            assert this.currentConfig == null;<a name="line.4385"></a>
<span class="sourceLineNo">4386</span>            assert this.clusterId == 0;<a name="line.4386"></a>
<span class="sourceLineNo">4387</span>            assert this.raftLog.isEmpty();<a name="line.4387"></a>
<span class="sourceLineNo">4388</span>            assert this.logDirChannel == null;<a name="line.4388"></a>
<span class="sourceLineNo">4389</span>            assert this.executor == null;<a name="line.4389"></a>
<span class="sourceLineNo">4390</span>            assert this.transmitting.isEmpty();<a name="line.4390"></a>
<span class="sourceLineNo">4391</span>            assert this.openTransactions.isEmpty();<a name="line.4391"></a>
<span class="sourceLineNo">4392</span>            assert this.pendingService.isEmpty();<a name="line.4392"></a>
<span class="sourceLineNo">4393</span>            assert !this.shuttingDown;<a name="line.4393"></a>
<span class="sourceLineNo">4394</span>            return;<a name="line.4394"></a>
<span class="sourceLineNo">4395</span>        }<a name="line.4395"></a>
<span class="sourceLineNo">4396</span><a name="line.4396"></a>
<span class="sourceLineNo">4397</span>        // Handle running state<a name="line.4397"></a>
<span class="sourceLineNo">4398</span>        assert this.kv != null;<a name="line.4398"></a>
<span class="sourceLineNo">4399</span>        assert this.random != null;<a name="line.4399"></a>
<span class="sourceLineNo">4400</span>        assert this.executor != null;<a name="line.4400"></a>
<span class="sourceLineNo">4401</span>        assert this.logDirChannel != null;<a name="line.4401"></a>
<span class="sourceLineNo">4402</span>        assert !this.executor.isShutdown() || this.shuttingDown;<a name="line.4402"></a>
<span class="sourceLineNo">4403</span><a name="line.4403"></a>
<span class="sourceLineNo">4404</span>        assert this.currentTerm &gt;= 0;<a name="line.4404"></a>
<span class="sourceLineNo">4405</span>        assert this.commitIndex &gt;= 0;<a name="line.4405"></a>
<span class="sourceLineNo">4406</span>        assert this.lastAppliedTerm &gt;= 0;<a name="line.4406"></a>
<span class="sourceLineNo">4407</span>        assert this.lastAppliedIndex &gt;= 0;<a name="line.4407"></a>
<span class="sourceLineNo">4408</span>        assert this.lastAppliedConfig != null;<a name="line.4408"></a>
<span class="sourceLineNo">4409</span>        assert this.currentConfig != null;<a name="line.4409"></a>
<span class="sourceLineNo">4410</span><a name="line.4410"></a>
<span class="sourceLineNo">4411</span>        assert this.currentTerm &gt;= this.lastAppliedTerm;<a name="line.4411"></a>
<span class="sourceLineNo">4412</span>        assert this.commitIndex &gt;= this.lastAppliedIndex;<a name="line.4412"></a>
<span class="sourceLineNo">4413</span>        assert this.commitIndex &lt;= this.lastAppliedIndex + this.raftLog.size();<a name="line.4413"></a>
<span class="sourceLineNo">4414</span>        long index = this.lastAppliedIndex;<a name="line.4414"></a>
<span class="sourceLineNo">4415</span>        long term = this.lastAppliedTerm;<a name="line.4415"></a>
<span class="sourceLineNo">4416</span>        for (LogEntry logEntry : this.raftLog) {<a name="line.4416"></a>
<span class="sourceLineNo">4417</span>            assert logEntry.getIndex() == index + 1;<a name="line.4417"></a>
<span class="sourceLineNo">4418</span>            assert logEntry.getTerm() &gt;= term;<a name="line.4418"></a>
<span class="sourceLineNo">4419</span>            index = logEntry.getIndex();<a name="line.4419"></a>
<span class="sourceLineNo">4420</span>            term = logEntry.getTerm();<a name="line.4420"></a>
<span class="sourceLineNo">4421</span>        }<a name="line.4421"></a>
<span class="sourceLineNo">4422</span><a name="line.4422"></a>
<span class="sourceLineNo">4423</span>        // Check configured vs. unconfigured<a name="line.4423"></a>
<span class="sourceLineNo">4424</span>        if (this.isConfigured()) {<a name="line.4424"></a>
<span class="sourceLineNo">4425</span>            assert this.clusterId != 0;<a name="line.4425"></a>
<span class="sourceLineNo">4426</span>            assert this.currentTerm &gt; 0;<a name="line.4426"></a>
<span class="sourceLineNo">4427</span>            assert this.lastAppliedTerm &gt;= 0;<a name="line.4427"></a>
<span class="sourceLineNo">4428</span>            assert this.lastAppliedIndex &gt;= 0;<a name="line.4428"></a>
<span class="sourceLineNo">4429</span>            assert !this.currentConfig.isEmpty();<a name="line.4429"></a>
<span class="sourceLineNo">4430</span>            assert this.currentConfig.equals(this.buildCurrentConfig());<a name="line.4430"></a>
<span class="sourceLineNo">4431</span>            assert this.getLastLogTerm() &gt; 0;<a name="line.4431"></a>
<span class="sourceLineNo">4432</span>            assert this.getLastLogIndex() &gt; 0;<a name="line.4432"></a>
<span class="sourceLineNo">4433</span>        } else {<a name="line.4433"></a>
<span class="sourceLineNo">4434</span>            assert this.lastAppliedTerm == 0;<a name="line.4434"></a>
<span class="sourceLineNo">4435</span>            assert this.lastAppliedIndex == 0;<a name="line.4435"></a>
<span class="sourceLineNo">4436</span>            assert this.lastAppliedConfig.isEmpty();<a name="line.4436"></a>
<span class="sourceLineNo">4437</span>            assert this.currentConfig.isEmpty();<a name="line.4437"></a>
<span class="sourceLineNo">4438</span>            assert this.raftLog.isEmpty();<a name="line.4438"></a>
<span class="sourceLineNo">4439</span>        }<a name="line.4439"></a>
<span class="sourceLineNo">4440</span><a name="line.4440"></a>
<span class="sourceLineNo">4441</span>        // Check role<a name="line.4441"></a>
<span class="sourceLineNo">4442</span>        assert this.role.checkState();<a name="line.4442"></a>
<span class="sourceLineNo">4443</span><a name="line.4443"></a>
<span class="sourceLineNo">4444</span>        // Check transactions<a name="line.4444"></a>
<span class="sourceLineNo">4445</span>        for (RaftKVTransaction tx : this.openTransactions.values())<a name="line.4445"></a>
<span class="sourceLineNo">4446</span>            tx.checkStateOpen(this.currentTerm, this.getLastLogIndex(), this.commitIndex);<a name="line.4446"></a>
<span class="sourceLineNo">4447</span>    }<a name="line.4447"></a>
<span class="sourceLineNo">4448</span>}<a name="line.4448"></a>
<span class="sourceLineNo">4449</span><a name="line.4449"></a>




























































</pre>
</div>
</body>
</html>
